Repository: incubator-reef Updated Branches: refs/heads/master bc9f5b54d -> 01a303d78
[REEF-476] Add IFileSystem This change adds the interface `Org.Apache.REEF.IO.FileSystem.IFileSystem`, a local implementation and tests. JIRA: [REEF-476](https://issues.apache.org/jira/browse/REEF-476) Pull Request: This closes #305 Author: Markus Weimer <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/01a303d7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/01a303d7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/01a303d7 Branch: refs/heads/master Commit: 01a303d7810da6ba350b3e243c8f9e12e4f54af9 Parents: bc9f5b5 Author: Markus Weimer <[email protected]> Authored: Mon Jul 13 16:02:21 2015 -0700 Committer: Julia Wang <[email protected]> Committed: Wed Jul 22 13:21:24 2015 -0700 ---------------------------------------------------------------------- .../Org.Apache.REEF.IO.Tests.csproj | 1 + .../TestLocalFileSystem.cs | 210 +++++++++++++++++++ .../FileSystem/IFileSystem.cs | 105 ++++++++++ .../FileSystem/Local/LocalFileSystem.cs | 88 ++++++++ .../Local/LocalFileSystemConfiguration.cs | 32 +++ .../Org.Apache.REEF.IO.csproj | 3 + 6 files changed, 439 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj index 33b53c2..66971a4 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj +++ b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj @@ -41,6 +41,7 @@ under the License. </ItemGroup> <ItemGroup> <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="TestLocalFileSystem.cs" /> <Compile Include="TestRandomDataSet.cs" /> </ItemGroup> <ItemGroup> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs new file mode 100644 index 0000000..c8eb6fb --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.IO; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Org.Apache.REEF.IO.FileSystem; +using Org.Apache.REEF.IO.FileSystem.Local; +using Org.Apache.REEF.Tang.Implementations.Tang; + +namespace Org.Apache.REEF.IO.Tests +{ + + /// <summary> + /// Tests for Org.Apache.REEF.IO.FileSystem.Local.LocalFileSystem + /// </summary> + [TestClass] + public sealed class TestLocalFileSystem + { + private const string TempFileName = "REEF.TestLocalFileSystem.tmp"; + private const byte TestByte = 123; + + [TestMethod] + public void TestCreateAndOpenAndDelete() + { + var fs = GetFileSystem(); + + + // Create a temp file + var tempFilePath = Path.Combine(Path.GetTempPath(), TempFileName); + var tempFileUri = new Uri(tempFilePath); + + using (var s = fs.Create(tempFileUri)) + { + s.WriteByte(TestByte); + } + + Assert.IsTrue(fs.Exists(tempFileUri)); + Assert.IsTrue(File.Exists(tempFilePath)); + + // Make sure it was read correctly + using (var s = fs.Open(tempFileUri)) + { + Assert.AreEqual(TestByte, s.ReadByte()); + } + using (var s = File.Open(tempFilePath, FileMode.Open)) + { + Assert.AreEqual(TestByte, s.ReadByte()); + } + + // Delete it + fs.Delete(tempFileUri); + + Assert.IsFalse(fs.Exists(tempFileUri)); + Assert.IsFalse(File.Exists(tempFilePath)); + } + + [TestMethod] + public void TestCopyFromLocal() + { + var fs = GetFileSystem(); + var sourceFilePath = Path.Combine(Path.GetTempPath(), TempFileName); + MakeLocalTestFile(sourceFilePath); + + var destinationFilePath = sourceFilePath + ".copy"; + if (File.Exists(destinationFilePath)) + { + File.Delete(destinationFilePath); + } + + var destinationUri = new Uri(destinationFilePath); + fs.CopyFromLocal(sourceFilePath, destinationUri); + TestRemoteFile(fs, destinationUri); + + fs.Delete(destinationUri); + Assert.IsFalse(fs.Exists(destinationUri)); + + File.Delete(sourceFilePath); + Assert.IsFalse(File.Exists(sourceFilePath)); + } + + [TestMethod] + public void TestCopyToLocal() + { + var fs = GetFileSystem(); + var sourceFilePath = Path.Combine(Path.GetTempPath(), TempFileName); + var sourceUri = new Uri(sourceFilePath); + var destinationFilePath = sourceFilePath + ".copy"; + if (File.Exists(destinationFilePath)) + { + File.Delete(destinationFilePath); + } + + MakeRemoteTestFile(fs, sourceUri); + fs.CopyToLocal(sourceUri, destinationFilePath); + TestLocalFile(destinationFilePath); + + fs.Delete(sourceUri); + Assert.IsFalse(fs.Exists(sourceUri)); + + File.Delete(destinationFilePath); + Assert.IsFalse(File.Exists(destinationFilePath)); + } + + [TestMethod] + public void TestCopy() + { + var fs = GetFileSystem(); + var sourcePath = Path.Combine(Path.GetTempPath(), TempFileName); + var sourceUri = new Uri(sourcePath); + var destinationUri = new Uri(sourcePath + ".copy"); + MakeRemoteTestFile(fs, sourceUri); + if (fs.Exists(destinationUri)) + { + fs.Delete(destinationUri); + } + Assert.IsFalse(fs.Exists(destinationUri)); + fs.Copy(sourceUri, destinationUri); + Assert.IsTrue(fs.Exists(destinationUri)); + TestRemoteFile(fs, destinationUri); + fs.Delete(destinationUri); + Assert.IsFalse(fs.Exists(destinationUri)); + fs.Delete(sourceUri); + Assert.IsFalse(fs.Exists(sourceUri)); + } + + [TestMethod] + public void TestGetChildren() + { + var fs = GetFileSystem(); + var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName)+"/"); + fs.CreateDirectory(directoryUri); + var fileUri = new Uri(directoryUri, "testfile"); + + MakeRemoteTestFile(fs, fileUri); + var fileUris = fs.GetChildren(directoryUri).ToList(); + foreach (var uri in fileUris) + { + TestRemoteFile(fs, uri); + } + + Assert.AreEqual(1, fileUris.Count); + Assert.AreEqual(fileUri, fileUris[0]); + fs.Delete(fileUri); + fs.DeleteDirectory(directoryUri); + } + + private IFileSystem GetFileSystem() + { + return TangFactory.GetTang() + .NewInjector(LocalFileSystemConfiguration.ConfigurationModule.Build()) + .GetInstance<IFileSystem>(); + } + + private void MakeRemoteTestFile(IFileSystem fs, Uri path) + { + if (fs.Exists(path)) + { + fs.Delete(path); + } + using (var s = fs.Create(path)) + { + s.WriteByte(TestByte); + } + } + + private void MakeLocalTestFile(string filePath) + { + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + using (var s = File.Create(filePath)) + { + s.WriteByte(TestByte); + } + } + + private void TestRemoteFile(IFileSystem fs, Uri path) + { + using (var s = fs.Open(path)) + { + Assert.AreEqual(TestByte, s.ReadByte()); + } + } + + private void TestLocalFile(string filePath) + { + using (var s = File.Open(filePath, FileMode.Open)) + { + Assert.AreEqual(TestByte, s.ReadByte()); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs new file mode 100644 index 0000000..2a5e5e6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; + +namespace Org.Apache.REEF.IO.FileSystem +{ + /// <summary> + /// A file system abstraction. + /// </summary> + public interface IFileSystem + { + /// <summary> + /// Opens the given URI for reading + /// </summary> + /// <param name="fileUri"></param> + /// <returns></returns> + /// <exception cref="IOException">If the URI couldn't be opened.</exception> + Stream Open(Uri fileUri); + + /// <summary> + /// Creates a new file under the given URI. + /// </summary> + /// <param name="fileUri"></param> + /// <returns></returns> + /// <exception cref="IOException">If the URI couldn't be created.</exception> + Stream Create(Uri fileUri); + + /// <summary> + /// Deletes the file under the given URI. + /// </summary> + /// <param name="fileUri"></param> + /// <exception cref="IOException"></exception> + void Delete(Uri fileUri); + + /// <summary> + /// Determines whether a file exists under the given URI. + /// </summary> + /// <param name="fileUri"></param> + /// <returns></returns> + bool Exists(Uri fileUri); + + /// <summary> + /// Copies the file referenced by sourceUri to destinationUri. + /// </summary> + /// <param name="sourceUri"></param> + /// <param name="destinationUri"></param> + /// <exception cref="IOException"></exception> + void Copy(Uri sourceUri, Uri destinationUri); + + /// <summary> + /// Copies the remote file to a local file. + /// </summary> + /// <param name="remoteFileUri"></param> + /// <param name="localName"></param> + /// <exception cref="IOException"></exception> + void CopyToLocal(Uri remoteFileUri, string localName); + + /// <summary> + /// Copies the specified file to the remote location. + /// </summary> + /// <param name="localFileName"></param> + /// <param name="remoteFileUri"></param> + /// <exception cref="IOException"></exception> + void CopyFromLocal(string localFileName, Uri remoteFileUri); + + /// <summary> + /// Creates a new directory. + /// </summary> + /// <param name="directoryUri"></param> + /// <exception cref="IOException"></exception> + void CreateDirectory(Uri directoryUri); + + /// <summary> + /// Deletes a directory. + /// </summary> + /// <param name="directoryUri"></param> + /// <exception cref="IOException"></exception> + void DeleteDirectory(Uri directoryUri); + + /// <summary> + /// Get the children on the given URI, if that refers to a directory. + /// </summary> + /// <param name="directoryUri"></param> + /// <returns></returns> + /// <exception cref="IOException"></exception> + IEnumerable<Uri> GetChildren(Uri directoryUri); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs new file mode 100644 index 0000000..408a83f --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.FileSystem.Local +{ + /// <summary> + /// An implementation of IFileSystem on the filesystem of the host. + /// </summary> + internal sealed class LocalFileSystem : IFileSystem + { + [Inject] + private LocalFileSystem() + { + } + + public Stream Open(Uri fileUri) + { + return File.Open(fileUri.LocalPath, FileMode.Open); + } + + public Stream Create(Uri fileUri) + { + return File.Open(fileUri.LocalPath, FileMode.Create); + } + + public void Delete(Uri fileUri) + { + File.Delete(fileUri.LocalPath); + } + + public bool Exists(Uri fileUri) + { + return File.Exists(fileUri.LocalPath); + } + + public void Copy(Uri sourceUri, Uri destinationUri) + { + File.Copy(sourceUri.LocalPath, destinationUri.LocalPath); + } + + public void CopyToLocal(Uri remoteFileUri, string localName) + { + File.Copy(remoteFileUri.LocalPath, localName); + } + + public void CopyFromLocal(string localFileName, Uri remoteFileUri) + { + File.Copy(localFileName, remoteFileUri.LocalPath); + } + + public void CreateDirectory(Uri directoryUri) + { + Directory.CreateDirectory(directoryUri.LocalPath); + } + + public void DeleteDirectory(Uri directoryUri) + { + Directory.Delete(directoryUri.LocalPath); + } + + public IEnumerable<Uri> GetChildren(Uri directoryUri) + { + var localPath = Path.GetFullPath(directoryUri.LocalPath); + return Directory.GetFileSystemEntries(localPath) + .Select(entry => new Uri(Path.Combine(localPath, entry))); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystemConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystemConfiguration.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystemConfiguration.cs new file mode 100644 index 0000000..9a43206 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystemConfiguration.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.IO.FileSystem.Local +{ + /// <summary> + /// ConfigurationModule for the local file system. + /// </summary> + public sealed class LocalFileSystemConfiguration : ConfigurationModuleBuilder + { + public static ConfigurationModule ConfigurationModule = new LocalFileSystemConfiguration() + .BindImplementation(GenericType<IFileSystem>.Class, GenericType<LocalFileSystem>.Class) + .Build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/01a303d7/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj index e02bac3..6007582 100644 --- a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj +++ b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj @@ -41,6 +41,9 @@ under the License. <Reference Include="System.Xml" /> </ItemGroup> <ItemGroup> + <Compile Include="FileSystem\IFileSystem.cs" /> + <Compile Include="FileSystem\Local\LocalFileSystem.cs" /> + <Compile Include="FileSystem\Local\LocalFileSystemConfiguration.cs" /> <Compile Include="PartitionedData\IPartition.cs" /> <Compile Include="PartitionedData\IPartitionDescriptor.cs" /> <Compile Include="PartitionedData\IPartitionedDataSet.cs" />
