Repository: incubator-reef Updated Branches: refs/heads/master 160801004 -> fe9f50878
[REEF-803] Add a ConfigurationProvider for HadoopFileSystem Added `HadoopFileSystemConfigurationProvider` and moved `DriverConfigurationProviders` to Common as it is shared by multiple providers. JIRA: [REEF-803](https://issues.apache.org/jira/browse/REEF-803) Pull Request: This closes #539 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/fe9f5087 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/fe9f5087 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/fe9f5087 Branch: refs/heads/master Commit: fe9f50878e6f3592a375548a593c00fa1164ce71 Parents: 1608010 Author: Julia Wang <[email protected]> Authored: Sat Oct 3 17:37:32 2015 -0700 Committer: Markus Weimer <[email protected]> Committed: Mon Oct 5 14:36:13 2015 -0700 ---------------------------------------------------------------------- .../API/JobSubmissionBuilderFactory.cs | 2 +- .../Parameters/DriverConfigurationProviders.cs | 33 ----------- .../API/TcpPortConfigurationModule.cs | 2 +- .../Org.Apache.REEF.Client.csproj | 1 - .../Parameters/DriverConfigurationProviders.cs | 34 ++++++++++++ .../Io/TcpPortConfigurationProvider.cs | 2 - .../Org.Apache.REEF.Common.csproj | 1 + .../OnREEFIMRURunTimeConfiguration.cs | 5 +- .../TestHadoopFileSystem.cs | 28 +++++++++- .../Hadoop/HadoopFileSystemConfiguration.cs | 9 +++ .../HadoopFileSystemConfigurationProvider.cs | 58 ++++++++++++++++++++ .../FileSystem/IFileSystem.cs | 3 + .../Org.Apache.REEF.IO.csproj | 1 + 13 files changed, 139 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs index c6811d9..c152927 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/JobSubmissionBuilderFactory.cs @@ -18,7 +18,7 @@ */ using System.Collections.Generic; -using Org.Apache.REEF.Client.API.Parameters; +using Org.Apache.REEF.Common.Client.Parameters; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Interface; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Client/API/Parameters/DriverConfigurationProviders.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/Parameters/DriverConfigurationProviders.cs b/lang/cs/Org.Apache.REEF.Client/API/Parameters/DriverConfigurationProviders.cs deleted file mode 100644 index 085e7a8..0000000 --- a/lang/cs/Org.Apache.REEF.Client/API/Parameters/DriverConfigurationProviders.cs +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using System.Collections.Generic; -using Org.Apache.REEF.Tang.Annotations; -using Org.Apache.REEF.Tang.Interface; - -namespace Org.Apache.REEF.Client.API.Parameters -{ - /// <summary> - // This name parameter is used to target receviers Configuration providers at driver level - /// </summary> - [NamedParameter] - public sealed class DriverConfigurationProviders : Name<ISet<IConfigurationProvider>> - { - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Client/API/TcpPortConfigurationModule.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/API/TcpPortConfigurationModule.cs b/lang/cs/Org.Apache.REEF.Client/API/TcpPortConfigurationModule.cs index b6fd4eb..affe511 100644 --- a/lang/cs/Org.Apache.REEF.Client/API/TcpPortConfigurationModule.cs +++ b/lang/cs/Org.Apache.REEF.Client/API/TcpPortConfigurationModule.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Org.Apache.REEF.Client.API.Parameters; using Org.Apache.REEF.Common.Attributes; using Org.Apache.REEF.Common.Io; +using Org.Apache.REEF.Common.Client.Parameters; using Org.Apache.REEF.Tang.Formats; using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Tang.Util; http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj index ff839b5..c261dfe 100644 --- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj +++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj @@ -62,7 +62,6 @@ under the License. <Compile Include="API\JobSubmission.cs" /> <Compile Include="API\JobSubmissionBuilder.cs" /> <Compile Include="API\JobSubmissionBuilderFactory.cs" /> - <Compile Include="API\Parameters\DriverConfigurationProviders.cs" /> <Compile Include="API\TcpPortConfigurationModule.cs" /> <Compile Include="Common\ClientConstants.cs" /> <Compile Include="Common\ClrClient2JavaClientCuratedParameters.cs" /> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Common/Client/Parameters/DriverConfigurationProviders.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Client/Parameters/DriverConfigurationProviders.cs b/lang/cs/Org.Apache.REEF.Common/Client/Parameters/DriverConfigurationProviders.cs new file mode 100644 index 0000000..4b55155 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Client/Parameters/DriverConfigurationProviders.cs @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.Common.Client.Parameters +{ + /// <summary> + // This name parameter is used to target receivers Configuration providers at driver level + /// </summary> + [NamedParameter] + public sealed class DriverConfigurationProviders : Name<ISet<IConfigurationProvider>> + { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Common/Io/TcpPortConfigurationProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Io/TcpPortConfigurationProvider.cs b/lang/cs/Org.Apache.REEF.Common/Io/TcpPortConfigurationProvider.cs index 16bc014..ec1c0f4 100644 --- a/lang/cs/Org.Apache.REEF.Common/Io/TcpPortConfigurationProvider.cs +++ b/lang/cs/Org.Apache.REEF.Common/Io/TcpPortConfigurationProvider.cs @@ -17,12 +17,10 @@ * under the License. */ -using System; using Org.Apache.REEF.Common.Evaluator.Parameters; using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; using Org.Apache.REEF.Tang.Interface; -using Org.Apache.REEF.Wake.Remote; using Org.Apache.REEF.Wake.Remote.Parameters; namespace Org.Apache.REEF.Common.Io http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj index 94c9758..f198e47 100644 --- a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj +++ b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj @@ -96,6 +96,7 @@ under the License. <Compile Include="Io\NamingConfigurationOptions.cs" /> <Compile Include="Io\TcpPortConfigurationProvider.cs" /> <Compile Include="ITaskSubmittable.cs" /> + <Compile Include="Client\Parameters\DriverConfigurationProviders.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Protobuf\ReefProtocol\ClientRuntime.pb.cs" /> <Compile Include="Protobuf\ReefProtocol\DriverRuntime.pb.cs" /> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs index 4502bfa..eaf99b2 100644 --- a/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/OnREEFIMRURunTimeConfiguration.cs @@ -21,6 +21,7 @@ using System.Globalization; using Org.Apache.REEF.Client.Local; using Org.Apache.REEF.Client.Yarn; using Org.Apache.REEF.IMRU.OnREEF.Client; +using Org.Apache.REEF.IO.FileSystem.Hadoop; using Org.Apache.REEF.Tang.Implementations.Configuration; using Org.Apache.REEF.Tang.Interface; @@ -72,9 +73,11 @@ namespace Org.Apache.REEF.IMRU.Examples IConfiguration imruClientConfig = REEFIMRUClientConfiguration<TMapInput, TMapOutput, TResult>.ConfigurationModule.Build(); + var fileSystemConfig = HadoopFileSystemConfiguration.ConfigurationModule.Build(); + IConfiguration runtimeConfig = YARNClientConfiguration.ConfigurationModule.Build(); - return Configurations.Merge(runtimeConfig, imruClientConfig); + return Configurations.Merge(runtimeConfig, imruClientConfig, fileSystemConfig); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs index 7e26231..ed0e222 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs @@ -19,7 +19,10 @@ using System; using System.IO; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Org.Apache.REEF.IO.FileSystem; using Org.Apache.REEF.IO.FileSystem.Hadoop; +using Org.Apache.REEF.IO.FileSystem.Hadoop.Parameters; +using Org.Apache.REEF.Tang.Annotations; using Org.Apache.REEF.Tang.Implementations.Tang; namespace Org.Apache.REEF.IO.Tests @@ -27,7 +30,7 @@ namespace Org.Apache.REEF.IO.Tests /// <summary> /// Tests for HadoopFileSystem. /// </summary> - /// <see cref="HadoopFileSystem" /> + /// <see cref="HadoopFileSystem" /> [TestClass] [Ignore] // These tests need to be run in an environment with HDFS installed. public sealed class TestHadoopFileSystem @@ -143,5 +146,28 @@ namespace Org.Apache.REEF.IO.Tests { _fileSystem.Create(GetTempUri()); } + + /// <summary> + /// This test is to make sure with the HadoopFileSystemConfiguration, HadoopFileSystem can be injected. + /// </summary> + [TestMethod] + public void TestHadoopFileSystemConfiguration() + { + var fileSystemTest = TangFactory.GetTang().NewInjector(HadoopFileSystemConfiguration.ConfigurationModule + .Build()) + .GetInstance<FileSystemTest>(); + Assert.IsTrue(fileSystemTest.FileSystem is HadoopFileSystem); + } + } + + class FileSystemTest + { + public IFileSystem FileSystem { get; private set; } + + [Inject] + private FileSystemTest(IFileSystem fileSystem) + { + FileSystem = fileSystem; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfiguration.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfiguration.cs index 0a9dcda..29b6ae1 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfiguration.cs @@ -15,8 +15,11 @@ // specific language governing permissions and limitations // under the License. +using Org.Apache.REEF.Common.Io; +using Org.Apache.REEF.Common.Client.Parameters; using Org.Apache.REEF.IO.FileSystem.Hadoop.Parameters; using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Interface; using Org.Apache.REEF.Tang.Util; namespace Org.Apache.REEF.IO.FileSystem.Hadoop @@ -46,7 +49,13 @@ namespace Org.Apache.REEF.IO.FileSystem.Hadoop /// </summary> public static readonly OptionalParameter<string> HadoopHome = new OptionalParameter<string>(); + /// <summary> + /// Set HadoopFileSystemConfigurationProvider to DriverConfigurationProviders. + /// Set all the parameters needed for injecting HadoopFileSystemConfigurationProvider. + /// </summary> public static readonly ConfigurationModule ConfigurationModule = new HadoopFileSystemConfiguration() + .BindSetEntry<DriverConfigurationProviders, HadoopFileSystemConfigurationProvider, IConfigurationProvider>( + GenericType<DriverConfigurationProviders>.Class, GenericType<HadoopFileSystemConfigurationProvider>.Class) .BindImplementation(GenericType<IFileSystem>.Class, GenericType<HadoopFileSystem>.Class) .BindNamedParameter(GenericType<NumberOfRetries>.Class, CommandRetries) .BindNamedParameter(GenericType<CommandTimeOut>.Class, CommandTimeOut) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfigurationProvider.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfigurationProvider.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfigurationProvider.cs new file mode 100644 index 0000000..84111b3 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystemConfigurationProvider.cs @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Common.Evaluator.Parameters; +using Org.Apache.REEF.IO.FileSystem.Hadoop.Parameters; +using Org.Apache.REEF.Tang.Annotations; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.IO.FileSystem.Hadoop +{ + /// <summary> + /// This provider provides configuration for HardoopFileSystem + /// The client that is going to use HadoopFileSystem in its driver and evaluators should set + /// configuration data through HadoopFileSystemConfiguration module in he client's configuration + /// </summary> + internal sealed class HadoopFileSystemConfigurationProvider : IConfigurationProvider + { + private readonly IConfiguration _configuration; + + [Inject] + private HadoopFileSystemConfigurationProvider( + [Parameter(typeof(NumberOfRetries))] int numberOfRetries, + [Parameter(typeof(CommandTimeOut))] int commandTimeOut, + [Parameter(typeof(HadoopHome))] string hadoopHome) + { + _configuration = TangFactory.GetTang().NewConfigurationBuilder() + .BindImplementation(GenericType<IFileSystem>.Class, GenericType<HadoopFileSystem>.Class) + .BindIntNamedParam<NumberOfRetries>(numberOfRetries.ToString()) + .BindIntNamedParam<CommandTimeOut>(commandTimeOut.ToString()) + .BindStringNamedParam<HadoopHome>(hadoopHome) + .BindSetEntry<EvaluatorConfigurationProviders, HadoopFileSystemConfigurationProvider, IConfigurationProvider>() + .Build(); + } + + IConfiguration IConfigurationProvider.GetConfiguration() + { + return _configuration; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs index 2a5e5e6..31f59fd 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs @@ -18,12 +18,15 @@ using System; using System.Collections.Generic; using System.IO; +using Org.Apache.REEF.IO.FileSystem.Local; +using Org.Apache.REEF.Tang.Annotations; namespace Org.Apache.REEF.IO.FileSystem { /// <summary> /// A file system abstraction. /// </summary> + [DefaultImplementation(typeof(LocalFileSystem), "default")] public interface IFileSystem { /// <summary> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/fe9f5087/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj index 37aef05..53d37e4 100644 --- a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj +++ b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj @@ -43,6 +43,7 @@ under the License. <ItemGroup> <Compile Include="FileSystem\Hadoop\CommandResult.cs" /> <Compile Include="FileSystem\Hadoop\HadoopFileSystemConfiguration.cs" /> + <Compile Include="FileSystem\Hadoop\HadoopFileSystemConfigurationProvider.cs" /> <Compile Include="FileSystem\Hadoop\HDFSCommandRunner.cs" /> <Compile Include="FileSystem\Hadoop\HadoopFileSystem.cs" /> <Compile Include="FileSystem\Hadoop\Parameters\CommandTimeOut.cs" />
