Author: hairong
Date: Thu Dec 9 22:13:06 2010
New Revision: 1044146
URL: http://svn.apache.org/viewvc?rev=1044146&view=rev
Log:
HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API. Contributed
by Patrick Kling.
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Thu Dec 9 22:13:06 2010
@@ -24,7 +24,10 @@ Trunk (unreleased changes)
(Patrick Kling via eli)
HADOOP-7054 Change NN LoadGenerator to use FileContext APIs
- (Sanjay Radia)
+ (Sanjay Radia)
+
+ HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API.
+ (Patrick Kling via hairong)
OPTIMIZATIONS
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
Thu Dec 9 22:13:06 2010
@@ -835,11 +835,11 @@ public abstract class AbstractFileSystem
UnresolvedLinkException, IOException;
/**
- * @return a list in which each entry describes a corrupt file/block
+ * @return an iterator over the corrupt files under the given path
+ * (may contain duplicates if a file has more than one corrupt block)
* @throws IOException
*/
- public CorruptFileBlocks listCorruptFileBlocks(String path,
- String cookie)
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
" does not support" +
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
Thu Dec 9 22:13:06 2010
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.Text;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Contains a list of paths corresponding to corrupt files and a cookie
- * used for iterative calls to NameNode.listCorruptFileBlocks.
- *
- */
-public class CorruptFileBlocks implements Writable {
- // used for hashCode
- private static final int PRIME = 16777619;
-
- private String[] files;
- private String cookie;
-
- public CorruptFileBlocks() {
- this(new String[0], "");
- }
-
- public CorruptFileBlocks(String[] files, String cookie) {
- this.files = files;
- this.cookie = cookie;
- }
-
- public String[] getFiles() {
- return files;
- }
-
- public String getCookie() {
- return cookie;
- }
-
- /**
- * {...@inheritdoc}
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- int fileCount = in.readInt();
- files = new String[fileCount];
- for (int i = 0; i < fileCount; i++) {
- files[i] = Text.readString(in);
- }
- cookie = Text.readString(in);
- }
-
- /**
- * {...@inheritdoc}
- */
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(files.length);
- for (int i = 0; i < files.length; i++) {
- Text.writeString(out, files[i]);
- }
- Text.writeString(out, cookie);
- }
-
- /**
- * {...@inheritdoc}
- */
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof CorruptFileBlocks)) {
- return false;
- }
- CorruptFileBlocks other = (CorruptFileBlocks) obj;
- return cookie.equals(other.cookie) &&
- Arrays.equals(files, other.files);
- }
-
- /**
- * {...@inheritdoc}
- */
- public int hashCode() {
- int result = cookie.hashCode();
-
- for (String file : files) {
- result = PRIME * result + file.hashCode();
- }
-
- return result;
- }
-}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Thu Dec
9 22:13:06 2010
@@ -1298,18 +1298,19 @@ public final class FileContext {
}
/**
- * @return a list in which each entry describes a corrupt file/block
+ * @return an iterator over the corrupt files under the given path
+ * (may contain duplicates if a file has more than one corrupt block)
* @throws IOException
*/
- public CorruptFileBlocks listCorruptFileBlocks(final String path,
- final String cookie)
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
- final Path absF = fixRelativePart(new Path(path));
- return new FSLinkResolver<CorruptFileBlocks>() {
+ final Path absF = fixRelativePart(path);
+ return new FSLinkResolver<RemoteIterator<Path>>() {
@Override
- public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p)
+ public RemoteIterator<Path> next(final AbstractFileSystem fs,
+ final Path p)
throws IOException, UnresolvedLinkException {
- return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+ return fs.listCorruptFileBlocks(p);
}
}.resolve(this, absF);
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Thu Dec
9 22:13:06 2010
@@ -1091,11 +1091,11 @@ public abstract class FileSystem extends
}
/**
- * @return a list in which each entry describes a corrupt file/block
+ * @return an iterator over the corrupt files under the given path
+ * (may contain duplicates if a file has more than one corrupt block)
* @throws IOException
*/
- public CorruptFileBlocks listCorruptFileBlocks(String path,
- String cookie)
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
" does not support" +
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Thu
Dec 9 22:13:06 2010
@@ -170,10 +170,9 @@ public class FilterFileSystem extends Fi
* {...@inheritdoc}
*/
@Override
- public CorruptFileBlocks listCorruptFileBlocks(String path,
- String cookie)
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
- return fs.listCorruptFileBlocks(path, cookie);
+ return fs.listCorruptFileBlocks(path);
}
/** List files and its block locations in a directory. */
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java Thu Dec 9
22:13:06 2010
@@ -168,10 +168,9 @@ public abstract class FilterFs extends A
* {...@inheritdoc}
*/
@Override
- public CorruptFileBlocks listCorruptFileBlocks(String path,
- String cookie)
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
- return myFs.listCorruptFileBlocks(path, cookie);
+ return myFs.listCorruptFileBlocks(path);
}
@Override
Modified:
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
---
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
(original)
+++
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
Thu Dec 9 22:13:06 2010
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.*;
-import org.junit.Test;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-
-public class TestCorruptFileBlocks {
-
- /**
- * Serialize the cfb given, deserialize and return the result.
- */
- static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb)
- throws IOException {
- DataOutputBuffer buf = new DataOutputBuffer();
- cfb.write(buf);
-
- byte[] data = buf.getData();
- DataInputStream input = new DataInputStream(new
ByteArrayInputStream(data));
-
- CorruptFileBlocks result = new CorruptFileBlocks();
- result.readFields(input);
-
- return result;
- }
-
- /**
- * Check whether cfb is unchanged after serialization and deserialization.
- */
- static boolean checkSerialize(CorruptFileBlocks cfb)
- throws IOException {
- return cfb.equals(serializeAndDeserialize(cfb));
- }
-
- /**
- * Test serialization and deserializaton of CorruptFileBlocks.
- */
- @Test
- public void testSerialization() throws IOException {
- {
- CorruptFileBlocks cfb = new CorruptFileBlocks();
- assertTrue(checkSerialize(cfb));
- }
-
- {
- String[] files = new String[0];
- CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
- assertTrue(checkSerialize(cfb));
- }
-
- {
- String[] files = { "a", "bb", "ccc" };
- CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
- assertTrue(checkSerialize(cfb));
- }
- }
-}
\ No newline at end of file
