Repository: hadoop Updated Branches: refs/heads/branch-2 f0324738c -> de7f9a8bc
HDFS-8079. Move CorruptFileBlockIterator to a new hdfs.client.impl package. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de7f9a8b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de7f9a8b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de7f9a8b Branch: refs/heads/branch-2 Commit: de7f9a8bcc2f5c68caed97c052c1b9ed8a1473e0 Parents: f032473 Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Wed Apr 8 11:50:52 2015 -0700 Committer: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Committed: Wed Apr 8 11:51:43 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../main/java/org/apache/hadoop/fs/Hdfs.java | 2 +- .../hadoop/hdfs/CorruptFileBlockIterator.java | 104 ------------------ .../hadoop/hdfs/DistributedFileSystem.java | 1 + .../client/impl/CorruptFileBlockIterator.java | 105 +++++++++++++++++++ .../namenode/TestListCorruptFileBlocks.java | 2 +- 6 files changed, 111 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e767f45..7736f62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -76,6 +76,9 @@ Release 2.8.0 - UNRELEASED HDFS-8080. Separate JSON related routines used by WebHdfsFileSystem to a package local class. (wheat9) + HDFS-8085. Move CorruptFileBlockIterator to a new hdfs.client.impl package. + (szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 8c09193..aaaff25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -35,13 +35,13 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.ChecksumOpt; -import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/CorruptFileBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/CorruptFileBlockIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/CorruptFileBlockIterator.java deleted file mode 100644 index 1597b87..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/CorruptFileBlockIterator.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs; - -import java.io.IOException; -import java.util.NoSuchElementException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; - -/** - * Provides an iterator interface for listCorruptFileBlocks. - * This class is used by DistributedFileSystem and Hdfs. - */ -@InterfaceAudience.Private -public class CorruptFileBlockIterator implements RemoteIterator<Path> { - private final DFSClient dfs; - private final String path; - - private String[] files = null; - private int fileIdx = 0; - private String cookie = null; - private Path nextPath = null; - - private int callsMade = 0; - - public CorruptFileBlockIterator(DFSClient dfs, Path path) throws IOException { - this.dfs = dfs; - this.path = path2String(path); - loadNext(); - } - - /** - * @return the number of calls made to the DFSClient. - * This is for debugging and testing purposes. - */ - public int getCallsMade() { - return callsMade; - } - - private String path2String(Path path) { - return path.toUri().getPath(); - } - - private Path string2Path(String string) { - return new Path(string); - } - - private void loadNext() throws IOException { - if (files == null || fileIdx >= files.length) { - CorruptFileBlocks cfb = dfs.listCorruptFileBlocks(path, cookie); - files = cfb.getFiles(); - cookie = cfb.getCookie(); - fileIdx = 0; - callsMade++; - } - - if (fileIdx >= files.length) { - // received an empty response - // there are no more corrupt file blocks - nextPath = null; - } else { - nextPath = string2Path(files[fileIdx]); - fileIdx++; - } - } - - - @Override - public boolean hasNext() { - return nextPath != null; - } - - - @Override - public Path next() throws IOException { - if (!hasNext()) { - throw new NoSuchElementException("No more corrupt file blocks"); - } - - Path result = nextPath; - loadNext(); - - return result; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 46b8a2a..ad41621 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -63,6 +63,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/CorruptFileBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/CorruptFileBlockIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/CorruptFileBlockIterator.java new file mode 100644 index 0000000..77bed1a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/CorruptFileBlockIterator.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.client.impl; + +import java.io.IOException; +import java.util.NoSuchElementException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; + +/** + * Provides an iterator interface for listCorruptFileBlocks. + * This class is used by DistributedFileSystem and Hdfs. + */ +@InterfaceAudience.Private +public class CorruptFileBlockIterator implements RemoteIterator<Path> { + private final DFSClient dfs; + private final String path; + + private String[] files = null; + private int fileIdx = 0; + private String cookie = null; + private Path nextPath = null; + + private int callsMade = 0; + + public CorruptFileBlockIterator(DFSClient dfs, Path path) throws IOException { + this.dfs = dfs; + this.path = path2String(path); + loadNext(); + } + + /** + * @return the number of calls made to the DFSClient. + * This is for debugging and testing purposes. + */ + public int getCallsMade() { + return callsMade; + } + + private String path2String(Path path) { + return path.toUri().getPath(); + } + + private Path string2Path(String string) { + return new Path(string); + } + + private void loadNext() throws IOException { + if (files == null || fileIdx >= files.length) { + CorruptFileBlocks cfb = dfs.listCorruptFileBlocks(path, cookie); + files = cfb.getFiles(); + cookie = cfb.getCookie(); + fileIdx = 0; + callsMade++; + } + + if (fileIdx >= files.length) { + // received an empty response + // there are no more corrupt file blocks + nextPath = null; + } else { + nextPath = string2Path(files[fileIdx]); + fileIdx++; + } + } + + + @Override + public boolean hasNext() { + return nextPath != null; + } + + + @Override + public Path next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException("No more corrupt file blocks"); + } + + Path result = nextPath; + loadNext(); + + return result; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/de7f9a8b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java index 7118b9e..3afdd0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java @@ -33,13 +33,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.BlockMissingException; -import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode;