[REEF-1950] Better test reporting for REEF.NET This adds an API to write functional tests of REEF. The main API is `ITestRunner` through which a Test can be submitted. In the Driver of a test, one can use an injected instance of `Assert` to record the passing or faling of tests.
A `ITestRunner` is obtained via the `TestRunnerFactory` which uses the same environment variables used by the Java tests to determine which TestRunner to instantiate. A good way to understand the new functionality is the new functional test, `TestTestFramework`. Known issues: * The `IAssert` API is spartan. * There is only an implementation for the local `TestRunner`. JIRA: [REEF-1950](https://issues.apache.org/jira/browse/REEF-1950) Pull Request: This closes #1412 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/3b89926a Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/3b89926a Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/3b89926a Branch: refs/heads/REEF-335 Commit: 3b89926a2498790b7056555acd098f42ac5d68a9 Parents: 3d571e6 Author: Markus Weimer <[email protected]> Authored: Sun Oct 22 09:55:17 2017 -0700 Committer: Sergiy Matusevych <[email protected]> Committed: Thu Nov 9 18:12:57 2017 -0800 ---------------------------------------------------------------------- .../Org.Apache.REEF.Client.Tests.csproj | 5 + .../TestFileWritingAssert.cs | 82 +++++++++ .../API/Testing/AbstractAssert.cs | 38 ++++ .../API/Testing/AssertResult.cs | 68 +++++++ .../API/Testing/IAssert.cs | 51 ++++++ .../API/Testing/ITestResult.cs | 48 +++++ .../API/Testing/ITestRunner.cs | 42 +++++ .../API/Testing/TestRunnerFactory.cs | 60 +++++++ .../FileWritingAssert/FileWritingAssert.cs | 47 +++++ .../FileWritingAssertConfiguration.cs | 39 +++++ .../TestRunner/FileWritingAssert/Parameters.cs | 30 ++++ .../TestRunner/FileWritingAssert/TestResult.cs | 175 +++++++++++++++++++ .../Local/TestRunner/LocalTestRunner.cs | 118 +++++++++++++ .../Org.Apache.REEF.Client.csproj | 11 ++ .../Functional/TestFramework/README.md | 3 + .../TestFramework/TestTestFramework.cs | 164 +++++++++++++++++ .../Org.Apache.REEF.Tests.csproj | 5 +- 17 files changed, 985 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj index 6674d25..f1bf43e 100644 --- a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj @@ -57,6 +57,7 @@ under the License. <Compile Include="LegacyJobResourceUploaderTests.cs" /> <Compile Include="MultipleRMUrlProviderTests.cs" /> <Compile Include="RestClientTests.cs" /> + <Compile Include="TestFileWritingAssert.cs" /> <Compile Include="WindowsHadoopEmulatorYarnClientTests.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="YarnClientTests.cs" /> @@ -89,6 +90,10 @@ under the License. <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project> <Name>Org.Apache.REEF.Wake</Name> </ProjectReference> + <ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.csproj"> + <Project>{545A0582-4105-44CE-B99C-B1379514A630}</Project> + <Name>Org.Apache.REEF.Common</Name> + </ProjectReference> </ItemGroup> <ItemGroup> <None Include="packages.config" /> http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client.Tests/TestFileWritingAssert.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/TestFileWritingAssert.cs b/lang/cs/Org.Apache.REEF.Client.Tests/TestFileWritingAssert.cs new file mode 100644 index 0000000..1f5d788 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client.Tests/TestFileWritingAssert.cs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert; +using Xunit; + +namespace Org.Apache.REEF.Client.Tests +{ + /// <summary> + /// Tests for the File Writing Assert + /// </summary> + public class TestFileWritingAssert + { + [Fact] + public void TestTestResult() + { + TestResult x = new TestResult(); + Assert.Equal(0, x.NumberOfPassedAsserts); + Assert.Equal(0, x.NumberOfFailedAsserts); + + x.Add(true, "Something went right"); + Assert.Equal(1, x.NumberOfPassedAsserts); + Assert.Equal(0, x.NumberOfFailedAsserts); + Assert.True(x.AllTestsSucceeded); + + x.IsTrue("Something else went right"); + Assert.Equal(2, x.NumberOfPassedAsserts); + Assert.Equal(0, x.NumberOfFailedAsserts); + Assert.True(x.AllTestsSucceeded); + + x.Add(false, "Something went wrong"); + Assert.Equal(2, x.NumberOfPassedAsserts); + Assert.Equal(1, x.NumberOfFailedAsserts); + Assert.False(x.AllTestsSucceeded); + + x.IsFalse("Something else went wrong"); + Assert.Equal(2, x.NumberOfPassedAsserts); + Assert.Equal(2, x.NumberOfFailedAsserts); + Assert.False(x.AllTestsSucceeded); + } + + [Fact] + public void TestTestResultFail() + { + var x = TestResult.Fail("OMG! It failed!"); + Assert.Equal(0, x.NumberOfPassedAsserts); + Assert.Equal(1, x.NumberOfFailedAsserts); + Assert.False(x.AllTestsSucceeded); + } + + [Fact] + public void TestTestResultSerialization() + { + TestResult before = new TestResult(); + before.Add(true, "Something went right"); + before.Add(true, "Something else went right"); + before.Add(false, "Something went wrong"); + + TestResult after = TestResult.FromJson(before.ToJson()); + + Assert.NotNull(after); + Assert.Equal(1, after.NumberOfFailedAsserts); + Assert.Equal(2, after.NumberOfPassedAsserts); + + Assert.Equal(before.ToJson(), after.ToJson()); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/AbstractAssert.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/AbstractAssert.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/AbstractAssert.cs new file mode 100644 index 0000000..608eb81 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/AbstractAssert.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <inheritdoc /> + /// <summary> + /// Helper class to ease the implementation of additional Assert classes. + /// </summary> + internal abstract class AbstractAssert : IAssert + { + public abstract void True(bool condition, string format, params object[] args); + + public void False(bool condition, string format, params object[] args) + { + True(!condition, format); + } + + public void Fail(string format, params object[] args) + { + True(false, format, args); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/AssertResult.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/AssertResult.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/AssertResult.cs new file mode 100644 index 0000000..7cb0a23 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/AssertResult.cs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Utilities.Attributes; +using System; +using System.Collections.Generic; + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <summary> + /// Serializable representation of a result of an assert. + /// </summary> + [Unstable("0.17", "Work in progress towards a new test infrastructure. See REEF-1271.")] + internal sealed class AssertResult : IEquatable<AssertResult> + { + public AssertResult(string message, bool isTrue) + { + Message = message; + IsTrue = isTrue; + } + + public string Message { get; } + + public bool IsTrue { get; } + + public bool IsFalse + { + get + { + return !IsTrue; + } + } + + public override bool Equals(object obj) + { + return Equals(obj as AssertResult); + } + + public bool Equals(AssertResult other) + { + return other != null && + Message == other.Message && + IsTrue == other.IsTrue; + } + + public override int GetHashCode() + { + var hashCode = -1707516999; + hashCode = (hashCode * -1521134295) + EqualityComparer<string>.Default.GetHashCode(Message); + hashCode = (hashCode * -1521134295) + IsTrue.GetHashCode(); + return hashCode; + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/IAssert.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/IAssert.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/IAssert.cs new file mode 100644 index 0000000..7f2ecd3 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/IAssert.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Utilities.Attributes; + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <summary> + /// Assert methods to be used in tests of REEF and REEF applications. + /// </summary> + [Unstable("0.17", "Work in progress towards a new test infrastructure. See REEF-1271.")] + public interface IAssert + { + /// <summary> + /// Assert that a boolean condition is true. + /// </summary> + /// <param name="condition">The condition. True indicates a passed test, false otherwise.</param> + /// <param name="format">The error message for the test if condition is false.</param> + /// <param name="args">Arguments to `format`.</param> + void True(bool condition, string format, params object[] args); + + /// <summary> + /// Assert that a boolean condition is false. + /// </summary> + /// <param name="condition">The condition. False indicates a passed test, true otherwise.</param> + /// <param name="format">The error message for the test if condition is true.</param> + /// <param name="args">Arguments to `format`.</param> + void False(bool condition, string format, params object[] args); + + /// <summary> + /// Record a failed test. + /// </summary> + /// <param name="format">The message for the failed test.</param> + /// <param name="args">Arguments to `format`.</param> + void Fail(string format, params object[] args); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestResult.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestResult.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestResult.cs new file mode 100644 index 0000000..839c919 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestResult.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Utilities.Attributes; + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <summary> + /// Represents a test result. + /// </summary> + [Unstable("0.17", "Work in progress towards a new test infrastructure. See REEF-1271.")] + public interface ITestResult + { + /// <summary> + /// The number of failed asserts in this test. + /// </summary> + int NumberOfFailedAsserts { get; } + + /// <summary> + /// The number of passed asserts in this test. + /// </summary> + int NumberOfPassedAsserts { get; } + + /// <summary> + /// True, if all asserts passed. + /// </summary> + bool AllTestsSucceeded { get; } + + /// <summary> + /// The error message to use if AllTestsSucceeded is false. + /// </summary> + string FailedTestMessage { get; } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestRunner.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestRunner.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestRunner.cs new file mode 100644 index 0000000..4cfcdbf --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/ITestRunner.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Utilities.Attributes; + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <summary> + /// Runs REEF integration tests and reports their results. + /// </summary> + [Unstable("0.17", "Work in progress towards a new test infrastructure. See REEF-1271.")] + public interface ITestRunner + { + /// <summary> + /// Create a new JobRequestBuilder. + /// </summary> + /// <remarks>This may pre-configure the job request for this test runner.</remarks> + /// <returns>A new JobRequestBuilder.</returns> + JobRequestBuilder NewJobRequestBuilder(); + + /// <summary> + /// Runs the given Job as a test. + /// </summary> + /// <param name="jobRequestBuilder">The job to run.</param> + /// <returns>The test results obtained.</returns> + ITestResult RunTest(JobRequestBuilder jobRequestBuilder); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/API/Testing/TestRunnerFactory.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Testing/TestRunnerFactory.cs b/lang/cs/Org.Apache.REEF.Client/API/Testing/TestRunnerFactory.cs new file mode 100644 index 0000000..5382f1d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/API/Testing/TestRunnerFactory.cs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Client.Local.TestRunner; +using Org.Apache.REEF.Utilities.Attributes; +using System; + +namespace Org.Apache.REEF.Client.API.Testing +{ + /// <summary> + /// Factory for TestRunner instances. + /// </summary> + /// <remarks> + /// This class follows the same approach as org.apache.reef.tests.TestEnvironmentFactory in Java. It reads the + /// same environment variables to decide which test runner to instantiate. + /// </remarks> + [Unstable("0.17", "Work in progress towards a new test infrastructure. See REEF-1271.")] + public sealed class TestRunnerFactory + { + // See `org.apache.reef.tests.TestEnvironmentFactory` in Java. + private const string TestOnYARNEnvironmentVariable = "REEF_TEST_YARN"; + + /// <summary> + /// Instantiates a TestRunner based on the environment variables. + /// </summary> + /// <returns>A TestRunner instance.</returns> + public static ITestRunner NewTestRunner() + { + if (RunOnYarn()) + { + throw new NotImplementedException("Running tests on YARN is not supported yet."); + } + return LocalTestRunner.GetLocalTestRunner(); + } + + /// <summary> + /// Check whether the tests are supposed to be run on YARN. + /// </summary> + /// <returns>True, if the tests are supposed to run on YARN.</returns> + private static bool RunOnYarn() + { + return bool.TryParse(Environment.GetEnvironmentVariable(TestOnYARNEnvironmentVariable), + out bool runOnYARN) && runOnYARN; + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssert.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssert.cs b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssert.cs new file mode 100644 index 0000000..c23dc8c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssert.cs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.IO; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Client.API.Testing; + +namespace Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert +{ + internal sealed class FileWritingAssert : AbstractAssert + { + private readonly TestResult _testResult = new TestResult(); + private readonly string _filePath; + + /// <param name="filePath">The path to the file where the assert results shall be written.</param> + [Inject] + internal FileWritingAssert([Parameter(typeof(Parameters.AssertFilePath))] string filePath) + { + _filePath = filePath; + } + + public override void True(bool condition, string format, params object[] args) + { + _testResult.Add(condition, format, args); + WriteAssertsFile(); + } + + private void WriteAssertsFile() + { + File.WriteAllText(_filePath, _testResult.ToJson()); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssertConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssertConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssertConfiguration.cs new file mode 100644 index 0000000..7de7cbe --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/FileWritingAssertConfiguration.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Client.API.Testing; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert +{ + /// <summary> + /// Configuration Module for the file writing assert validation. + /// </summary> + internal sealed class FileWritingAssertConfiguration : ConfigurationModuleBuilder + { + /// <summary> + /// Path to the file to be written for the asserts. Uses a temp file if not set. + /// </summary> + public static readonly OptionalParameter<string> FilePath = new OptionalParameter<string>(); + + public static ConfigurationModule ConfigurationModule = new FileWritingAssertConfiguration() + .BindImplementation(GenericType<IAssert>.Class, GenericType<FileWritingAssert>.Class) + .BindNamedParameter(GenericType<Parameters.AssertFilePath>.Class, FilePath) + .Build(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/Parameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/Parameters.cs b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/Parameters.cs new file mode 100644 index 0000000..d83b25b --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/Parameters.cs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Tang.Annotations; + +// ReSharper disable once CheckNamespace +namespace Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert.Parameters +{ + [NamedParameter(documentation: "Path where the assert log shall be written.")] + internal sealed class AssertFilePath : Name<string> + { + private AssertFilePath() + { + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/TestResult.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/TestResult.cs b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/TestResult.cs new file mode 100644 index 0000000..2a3a16b --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/FileWritingAssert/TestResult.cs @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Collections.Generic; +using System.Linq; +using System; +using Newtonsoft.Json; +using Org.Apache.REEF.Client.API.Testing; + +namespace Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert +{ + internal sealed class TestResult : ITestResult, IEquatable<TestResult> + { + private readonly IList<AssertResult> _results; + + public TestResult(IEnumerable<AssertResult> results) + { + _results = results.ToList(); + } + + public TestResult() : this(new List<AssertResult>()) + { + // Intentionally empty + } + + /// <inheritdoc /> + public int NumberOfFailedAsserts + { + get + { + return _results.Count(_ => _.IsFalse); + } + } + + /// <inheritdoc /> + public int NumberOfPassedAsserts + { + get + { + return _results.Count(_ => _.IsTrue); + } + } + + /// <inheritdoc /> + public bool AllTestsSucceeded + { + get + { + return NumberOfFailedAsserts == 0; + } + } + + private IEnumerable<AssertResult> FailedAsserts + { + get + { + return _results.Where(_ => _.IsFalse); + } + } + + private IEnumerable<AssertResult> PassedAsserts + { + get + { + return _results.Where(_ => _.IsTrue); + } + } + + /// <inheritdoc /> + public string FailedTestMessage + { + get { return "Failed tests: " + string.Join(";", FailedAsserts.Select(_ => _.Message)); } + } + + /// <summary> + /// Add a Assert result to the collection. + /// </summary> + /// <param name="condition">Whether or not the condition was met.</param> + /// <param name="format">The message of the assert.</param> + /// <param name="args">Parameters to `format`.</param> + /// <returns>this, for chain calls</returns> + public TestResult Add(bool condition, string format, params object[] args) + { + _results.Add(new AssertResult(format, condition)); + return this; + } + + /// <summary> + /// Record an assert that passed. + /// </summary> + /// <param name="format">The message to record</param> + /// <param name="args">Parameters for the format string.</param> + /// <returns>this, for chain calls</returns> + public TestResult IsTrue(string format, params object[] args) + { + return Add(true, format, args); + } + + /// <summary> + /// Record an assert that failed. + /// </summary> + /// <param name="format">The message to record</param> + /// <param name="args">Parameters for the format string.</param> + /// <returns>this, for chain calls</returns> + public TestResult IsFalse(string format, params object[] args) + { + return Add(false, format, args); + } + + /// <summary> + /// Serializes the data contained in this object to JSON. + /// </summary> + /// <returns>A string version of this object.</returns> + public string ToJson() + { + return JsonConvert.SerializeObject(_results); + } + + /// <summary> + /// Deserializes an instance from a string generated by ToJson(). + /// </summary> + /// <param name="serializedObject">The object to deserialize.</param> + /// <returns>The deserialized object or null if serializedObject is null or whitespace.</returns> + public static TestResult FromJson(string serializedObject) + { + if (string.IsNullOrWhiteSpace(serializedObject)) + { + return null; + } + + return new TestResult(JsonConvert.DeserializeObject<List<AssertResult>>(serializedObject)); + } + + /// <summary> + /// Creates a TestResult with a single failure inside. + /// </summary> + /// <param name="format">The message for the failure.</param> + /// <param name="args">Parameters, if `format` refers to them.</param> + /// <returns>A TestResult with a single failure inside.</returns> + public static TestResult Fail(string format, params object[] args) + { + return new TestResult().IsFalse(format, args); + } + + public override bool Equals(object obj) + { + return Equals(obj as TestResult); + } + + public bool Equals(TestResult other) + { + return other != null && + EqualityComparer<IList<AssertResult>>.Default.Equals(_results, other._results); + } + + public override int GetHashCode() + { + return -3177284 + EqualityComparer<IList<AssertResult>>.Default.GetHashCode(_results); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/LocalTestRunner.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/LocalTestRunner.cs b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/LocalTestRunner.cs new file mode 100644 index 0000000..653917d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Client/Local/TestRunner/LocalTestRunner.cs @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.IO; +using Org.Apache.REEF.Client.API; +using Org.Apache.REEF.Client.Common; +using Org.Apache.REEF.Client.Local.TestRunner.FileWritingAssert; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Utilities.Logging; +using Org.Apache.REEF.Client.API.Testing; + +namespace Org.Apache.REEF.Client.Local.TestRunner +{ + /// <summary> + /// Runs a test on the local runtime. + /// </summary> + internal sealed class LocalTestRunner : ITestRunner + { + private static readonly Logger LOG = Logger.GetLogger(typeof(LocalTestRunner)); + private readonly IREEFClient _client; + + [Inject] + private LocalTestRunner(IREEFClient client) + { + _client = client; + } + + public JobRequestBuilder NewJobRequestBuilder() + { + return _client.NewJobRequestBuilder(); + } + + public ITestResult RunTest(JobRequestBuilder jobRequestBuilder) + { + // Setup the assert file. + var assertFileName = Path.GetTempPath() + "/reef-test-" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".json"; + jobRequestBuilder.AddDriverConfiguration(FileWritingAssertConfiguration.ConfigurationModule + .Set(FileWritingAssertConfiguration.FilePath, assertFileName) + .Build()); + var jobRequest = jobRequestBuilder.Build(); + + LOG.Log(Level.Info, "Submitting job `{0}` for execution. Assert log in `{1}`", + jobRequest.JobIdentifier, + assertFileName); + IJobSubmissionResult jobStatus = _client.SubmitAndGetJobStatus(jobRequest); + + if (null == jobStatus) + { + return TestResult.Fail( + "JobStatus returned by the Client was null. This points to an environment setup problem."); + } + + LOG.Log(Level.Verbose, "Waiting for job `{0}` to complete.", jobRequest.JobIdentifier); + jobStatus.WaitForDriverToFinish(); + LOG.Log(Level.Verbose, "Job `{0}` completed.", jobRequest.JobIdentifier); + + return ReadTestResult(assertFileName); + } + + private static TestResult ReadTestResult(string assertFilePath) + { + if (!File.Exists(assertFilePath)) + { + return TestResult.Fail("Test Results file {0} does not exist.", assertFilePath); + } + + try + { + return TestResult.FromJson(File.ReadAllText(assertFilePath)) + ?? TestResult.Fail("Results read from `{0}` where null.", assertFilePath); + } + catch (Exception exception) + { + return TestResult.Fail("Could not parse test results: {0}", exception); + } + } + + /// <summary> + /// Convenience method to generate a local test runner with the given number of containers. + /// </summary> + /// <param name="numberOfContainers"></param> + /// <returns></returns> + public static ITestRunner GetLocalTestRunner(int numberOfContainers = 4) + { + return GetLocalTestRunner( + LocalRuntimeClientConfiguration.ConfigurationModule + .Set(LocalRuntimeClientConfiguration.NumberOfEvaluators, numberOfContainers.ToString()) + .Build()); + } + + /// <summary> + /// Convenience method to instantiate a local test runner with the given runtime Configuration. + /// </summary> + /// <param name="runtimeConfiguration"></param> + /// <returns></returns> + public static ITestRunner GetLocalTestRunner(IConfiguration runtimeConfiguration) + { + return TangFactory.GetTang().NewInjector(runtimeConfiguration).GetInstance<LocalTestRunner>(); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index 879b8c3..451597f 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -69,6 +69,11 @@ under the License. <Compile Include="API\Exceptions\ClasspathException.cs" /> <Compile Include="API\Exceptions\JavaNotFoundException.cs" /> <Compile Include="API\IREEFClient.cs" /> + <Compile Include="API\Testing\AbstractAssert.cs" /> + <Compile Include="API\Testing\IAssert.cs" /> + <Compile Include="API\Testing\AssertResult.cs" /> + <Compile Include="API\Testing\ITestResult.cs" /> + <Compile Include="API\Testing\ITestRunner.cs" /> <Compile Include="API\JobParameters.cs" /> <Compile Include="API\JobParametersBuilder.cs" /> <Compile Include="API\JobRequest.cs" /> @@ -101,7 +106,13 @@ under the License. <Compile Include="Local\LocalRuntimeClientConfiguration.cs" /> <Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" /> <Compile Include="Local\Parameters\NumberOfEvaluators.cs" /> + <Compile Include="Local\TestRunner\FileWritingAssert\FileWritingAssert.cs" /> + <Compile Include="Local\TestRunner\FileWritingAssert\FileWritingAssertConfiguration.cs" /> + <Compile Include="Local\TestRunner\FileWritingAssert\Parameters.cs" /> + <Compile Include="Local\TestRunner\FileWritingAssert\TestResult.cs" /> + <Compile Include="Local\TestRunner\LocalTestRunner.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="API\Testing\TestRunnerFactory.cs" /> <Compile Include="YARN\ApplicationReport.cs" /> <Compile Include="YARN\Environment.cs" /> <Compile Include="YARN\HDI\HDInsightClientConfiguration.cs" /> http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/README.md ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/README.md b/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/README.md new file mode 100644 index 0000000..38910bb --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/README.md @@ -0,0 +1,3 @@ +# Test framework tests + +Tests in this namespace test the function of the test framework itself, not REEF. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/TestTestFramework.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/TestTestFramework.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/TestTestFramework.cs new file mode 100644 index 0000000..fa8e3d9 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Tests/Functional/TestFramework/TestTestFramework.cs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using Org.Apache.REEF.Driver; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; +using Xunit; +using Org.Apache.REEF.Driver.Evaluator; +using Org.Apache.REEF.Client.API.Testing; + +namespace Org.Apache.REEF.Tests.Functional.TestFramework +{ + /// <summary> + /// Tests of REEF's test framework + /// </summary> + public sealed class TestTestFramework + { + /// <summary> + /// Tests whether a Driver with a single failing assert is reported correctly. + /// </summary> + [Fact] + public void TestTestFailure() + { + ITestRunner testRunner = TestRunnerFactory.NewTestRunner(); + + // The TestRunner cannot be null. + Xunit.Assert.NotNull(testRunner); + + // Submit the job. + ITestResult testResult = testRunner.RunTest(testRunner.NewJobRequestBuilder() + .AddDriverConfiguration(TestFailingStartHandler.GetDriverConfiguration()) + .AddGlobalAssemblyForType(typeof(TestFailingStartHandler)) + .SetJobIdentifier("TestFailingTest")); + + // The TestResult cannot be null. + Xunit.Assert.NotNull(testResult); + + // There should be at least 1 failing assert. + Xunit.Assert.False(testResult.AllTestsSucceeded, testResult.FailedTestMessage); + + // Only the expected assert should have failed. + Xunit.Assert.Equal(1, testResult.NumberOfFailedAsserts); + } + + /// <summary> + /// Tests whether a Driver with a single passing test is reported correctly. + /// </summary> + [Fact] + public void TestTestPassing() + { + ITestRunner testRunner = TestRunnerFactory.NewTestRunner(); + + // The TestRunner cannot be null. + Xunit.Assert.NotNull(testRunner); + + // Submit the job. + ITestResult testResult = testRunner.RunTest(testRunner.NewJobRequestBuilder() + .AddDriverConfiguration(TestPassingStartHandler.GetDriverConfiguration()) + .AddGlobalAssemblyForType(typeof(TestPassingStartHandler)) + .SetJobIdentifier("TestPassingTest")); + + // The TestResult cannot be null. + Xunit.Assert.NotNull(testResult); + + // The TestResult cannot contain a failed assert. + Xunit.Assert.True(testResult.AllTestsSucceeded, testResult.FailedTestMessage); + + // The TestResult cannot contain more than one passed assert. + Xunit.Assert.Equal(1, testResult.NumberOfPassedAsserts); + } + } + + /// <inheritdoc /> + /// <summary> + /// A mock test which always fails. + /// </summary> + internal sealed class TestFailingStartHandler : IObserver<IDriverStarted> + { + private readonly Client.API.Testing.IAssert _assert; + + private const string FailedAssertMessage = "This test should never pass."; + + [Inject] + private TestFailingStartHandler(Client.API.Testing.IAssert assert, IEvaluatorRequestor evaluatorRequestor) + { + _assert = assert; + } + + public void OnNext(IDriverStarted value) + { + // Fail the test case. + _assert.True(false, FailedAssertMessage); + } + + public void OnError(Exception error) + { + _assert.True(false, "Call to OnError() received."); + } + + public void OnCompleted() + { + // empty on purpose. + } + + public static IConfiguration GetDriverConfiguration() + { + return DriverConfiguration.ConfigurationModule + .Set(DriverConfiguration.OnDriverStarted, GenericType<TestFailingStartHandler>.Class) + .Build(); + } + } + + /// <summary> + /// A mock test which always succeeds. + /// </summary> + internal sealed class TestPassingStartHandler : IObserver<IDriverStarted> + { + private readonly Client.API.Testing.IAssert _assert; + + [Inject] + private TestPassingStartHandler(Client.API.Testing.IAssert assert) + { + _assert = assert; + } + + public void OnNext(IDriverStarted value) + { + _assert.True(true, "This test should always pass."); + } + + public void OnError(Exception error) + { + _assert.True(false, "Call to OnError() received."); + } + + public void OnCompleted() + { + // empty on purpose. + } + + public static IConfiguration GetDriverConfiguration() + { + return DriverConfiguration.ConfigurationModule + .Set(DriverConfiguration.OnDriverStarted, GenericType<TestPassingStartHandler>.Class) + .Build(); + } + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/3b89926a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj index b83b705..ff71bd6 100644 --- a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj @@ -165,6 +165,7 @@ under the License. <Compile Include="Functional\Telemetry\MetricsDriver.cs" /> <Compile Include="Functional\Telemetry\MetricsTask.cs" /> <Compile Include="Functional\Telemetry\TestMetricsMessage.cs" /> + <Compile Include="Functional\TestFramework\TestTestFramework.cs" /> <Compile Include="Performance\TestHelloREEF\TestHelloDriver.cs" /> <Compile Include="Performance\TestHelloREEF\TestHelloREEFClient.cs" /> <Compile Include="Performance\TestHelloREEF\TestHelloTask.cs" /> @@ -248,7 +249,9 @@ under the License. <ItemGroup> <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" /> </ItemGroup> - <ItemGroup /> + <ItemGroup> + <None Include="Functional\TestFramework\README.md" /> + </ItemGroup> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" /> <Import Project="$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets" Condition="Exists('$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets')" />
