Author: arp
Date: Wed Nov 6 06:14:13 2013
New Revision: 1539245
URL: http://svn.apache.org/r1539245
Log:
Merging r1538408 through r1539244 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1538408-1539244
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Nov 6 06:14:13 2013
@@ -495,6 +495,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5257. addBlock() retry should return LocatedBlock with locations else
client
will get AIOBE. (Vinay via jing9)
+ HDFS-5427. Not able to read deleted files from snapshot directly under
+ snapshottable dir after checkpoint and NN restart. (Vinay via jing9)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -579,6 +582,12 @@ Release 2.2.1 - UNRELEASED
HDFS-5035. getFileLinkStatus and rename do not correctly check permissions
of symlinks. (Andrew Wang via Colin Patrick McCabe)
+ HDFS-5456. NameNode startup progress creates new steps if caller attempts
to
+ create a counter for a step that doesn't already exist. (cnauroth)
+
+ HDFS-5458. Datanode failed volume threshold ignored if exception is thrown
+ in getDataDirsFromURIs. (Mike Mellenthin via wang)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1538408-1539244
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Wed Nov 6 06:14:13 2013
@@ -1764,7 +1764,7 @@ public class DataNode extends Configured
} catch (IOException ioe) {
LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " "
+ location.getFile() + " : ", ioe);
-
invalidDirs.append("\"").append(location.getFile().getCanonicalPath()).append("\"
");
+
invalidDirs.append("\"").append(location.getUri().getPath()).append("\" ");
}
}
if (locations.size() == 0) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
Wed Nov 6 06:14:13 2013
@@ -588,8 +588,12 @@ public class FSImageFormat {
namesystem.dir.cacheName(child);
if (child.isFile()) {
+ updateBlocksMap(child.asFile());
+ }
+ }
+
+ public void updateBlocksMap(INodeFile file) {
// Add file->block mapping
- final INodeFile file = child.asFile();
final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
final BlockManager bm = namesystem.getBlockManager();
@@ -598,7 +602,6 @@ public class FSImageFormat {
}
}
}
- }
/** @return The FSDirectory of the namesystem where the fsimage is loaded
*/
public FSDirectory getFSDirectoryInLoading() {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
Wed Nov 6 06:14:13 2013
@@ -203,6 +203,9 @@ public class SnapshotFSImageFormat {
// useful, but set the parent here to be consistent with the original
// fsdir tree.
deleted.setParent(parent);
+ if (deleted.isFile()) {
+ loader.updateBlocksMap(deleted.asFile());
+ }
}
return deletedList;
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
Wed Nov 6 06:14:13 2013
@@ -149,8 +149,8 @@ public class StartupProgress {
* @return Counter associated with phase and step
*/
public Counter getCounter(Phase phase, Step step) {
- final StepTracking tracking = lazyInitStep(phase, step);
if (!isComplete()) {
+ final StepTracking tracking = lazyInitStep(phase, step);
return new Counter() {
@Override
public void increment() {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
Wed Nov 6 06:14:13 2013
@@ -21,6 +21,7 @@ import static org.apache.hadoop.test.Gen
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -205,4 +208,70 @@ public class TestSnapshotBlocksMap {
assertExceptionContains("File does not exist: " + s1f0, e);
}
}
+
+ /*
+ * Try to read the files inside snapshot but deleted in original place after
+ * restarting post checkpoint. refer HDFS-5427
+ */
+ @Test(timeout = 30000)
+ public void testReadSnapshotFileWithCheckpoint() throws Exception {
+ Path foo = new Path("/foo");
+ hdfs.mkdirs(foo);
+ hdfs.allowSnapshot(foo);
+ Path bar = new Path("/foo/bar");
+ DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
+ hdfs.createSnapshot(foo, "s1");
+ assertTrue(hdfs.delete(bar, true));
+
+ // checkpoint
+ NameNode nameNode = cluster.getNameNode();
+ NameNodeAdapter.enterSafeMode(nameNode, false);
+ NameNodeAdapter.saveNamespace(nameNode);
+ NameNodeAdapter.leaveSafeMode(nameNode);
+
+ // restart namenode to load snapshot files from fsimage
+ cluster.restartNameNode(true);
+ String snapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar");
+ DFSTestUtil.readFile(hdfs, new Path(snapshotPath));
+ }
+
+ /*
+ * Try to read the files inside snapshot but renamed to different file and
+ * deleted after restarting post checkpoint. refer HDFS-5427
+ */
+ @Test(timeout = 30000)
+ public void testReadRenamedSnapshotFileWithCheckpoint() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path foo2 = new Path("/foo2");
+ hdfs.mkdirs(foo);
+ hdfs.mkdirs(foo2);
+
+ hdfs.allowSnapshot(foo);
+ hdfs.allowSnapshot(foo2);
+ final Path bar = new Path(foo, "bar");
+ final Path bar2 = new Path(foo2, "bar");
+ DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
+ hdfs.createSnapshot(foo, "s1");
+ // rename to another snapshottable directory and take snapshot
+ assertTrue(hdfs.rename(bar, bar2));
+ hdfs.createSnapshot(foo2, "s2");
+ // delete the original renamed file to make sure blocks are not updated by
+ // the original file
+ assertTrue(hdfs.delete(bar2, true));
+
+ // checkpoint
+ NameNode nameNode = cluster.getNameNode();
+ NameNodeAdapter.enterSafeMode(nameNode, false);
+ NameNodeAdapter.saveNamespace(nameNode);
+ NameNodeAdapter.leaveSafeMode(nameNode);
+ // restart namenode to load snapshot files from fsimage
+ cluster.restartNameNode(true);
+ // file in first snapshot
+ String barSnapshotPath = Snapshot.getSnapshotPath(foo.toString(),
"s1/bar");
+ DFSTestUtil.readFile(hdfs, new Path(barSnapshotPath));
+ // file in second snapshot after rename+delete
+ String bar2SnapshotPath = Snapshot.getSnapshotPath(foo2.toString(),
+ "s2/bar");
+ DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath));
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java?rev=1539245&r1=1539244&r2=1539245&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
Wed Nov 6 06:14:13 2013
@@ -179,6 +179,14 @@ public class TestStartupProgress {
startupProgress.endStep(LOADING_FSIMAGE, step);
startupProgress.endPhase(LOADING_FSIMAGE);
+ // Also attempt a whole new step that wasn't used last time.
+ startupProgress.beginPhase(LOADING_EDITS);
+ Step newStep = new Step("file1");
+ startupProgress.beginStep(LOADING_EDITS, newStep);
+ incrementCounter(startupProgress, LOADING_EDITS, newStep, 100L);
+ startupProgress.endStep(LOADING_EDITS, newStep);
+ startupProgress.endPhase(LOADING_EDITS);
+
StartupProgressView after = startupProgress.createView();
// Expect that data was frozen after completion of entire startup process,
so
@@ -200,6 +208,7 @@ public class TestStartupProgress {
after.getTotal(LOADING_FSIMAGE));
assertEquals(before.getTotal(LOADING_FSIMAGE, step),
after.getTotal(LOADING_FSIMAGE, step));
+ assertFalse(after.getSteps(LOADING_EDITS).iterator().hasNext());
}
@Test(timeout=10000)
