Author: wheat9
Date: Fri Jun 20 18:56:00 2014
New Revision: 1604244
URL: http://svn.apache.org/r1604244
Log:
HDFS-6557. Merge r1604242 from trunk.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Jun 20 18:56:00 2014
@@ -202,6 +202,8 @@ Release 2.5.0 - UNRELEASED
HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun
Zhang via atm)
+ HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Fri Jun 20 18:56:00 2014
@@ -118,7 +118,6 @@ public class FSDirectory implements Clos
public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING);
INodeDirectory rootDir;
- FSImage fsImage;
private final FSNamesystem namesystem;
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
private final int maxComponentLength;
@@ -171,11 +170,10 @@ public class FSDirectory implements Clos
*/
private final NameCache<ByteArray> nameCache;
- FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
+ FSDirectory(FSNamesystem ns, Configuration conf) {
this.dirLock = new ReentrantReadWriteLock(true); // fair
rootDir = createRoot(ns);
inodeMap = INodeMap.newInstance(rootDir);
- this.fsImage = fsImage;
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
this.lsLimit = configuredLimit>0 ?
@@ -232,9 +230,7 @@ public class FSDirectory implements Clos
* Shutdown the filestore
*/
@Override
- public void close() throws IOException {
- fsImage.close();
- }
+ public void close() throws IOException {}
void markNameCacheInitialized() {
writeLock();
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Fri Jun 20 18:56:00 2014
@@ -495,7 +495,6 @@ public class FSImage implements Closeabl
FSImage realImage = target.getFSImage();
FSImage ckptImage = new FSImage(conf,
checkpointDirs, checkpointEditsDirs);
- target.dir.fsImage = ckptImage;
// load from the checkpoint dirs
try {
ckptImage.recoverTransitionRead(StartupOption.REGULAR, target, null);
@@ -507,7 +506,6 @@ public class FSImage implements Closeabl
realImage.getEditLog().setNextTxId(ckptImage.getEditLog().getLastWrittenTxId()+1);
realImage.initEditLog(StartupOption.IMPORT);
- target.dir.fsImage = realImage;
realImage.getStorage().setBlockPoolID(ckptImage.getBlockPoolID());
// and save it but keep the same checkpointTime
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Fri Jun 20 18:56:00 2014
@@ -506,6 +506,9 @@ public class FSNamesystem implements Nam
private volatile boolean imageLoaded = false;
private final Condition cond;
+
+ private final FSImage fsImage;
+
/**
* Notify that loading of this FSDirectory is complete, and
* it is imageLoaded for use
@@ -727,6 +730,7 @@ public class FSNamesystem implements Nam
LOG.info("fsLock is fair:" + fair);
fsLock = new FSNamesystemLock(fair);
cond = fsLock.writeLock().newCondition();
+ this.fsImage = fsImage;
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -818,7 +822,7 @@ public class FSNamesystem implements Nam
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
this.dtSecretManager = createDelegationTokenSecretManager(conf);
- this.dir = new FSDirectory(fsImage, this, conf);
+ this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
this.cacheManager = new CacheManager(this, conf, blockManager);
this.safeMode = new SafeModeInfo(conf);
@@ -1046,7 +1050,7 @@ public class FSNamesystem implements Nam
LOG.info("Starting services required for active state");
writeLock();
try {
- FSEditLog editLog = dir.fsImage.getEditLog();
+ FSEditLog editLog = getFSImage().getEditLog();
if (!editLog.isOpenForWrite()) {
// During startup, we're already open for write during initialization.
@@ -1075,12 +1079,12 @@ public class FSNamesystem implements Nam
metaSaveAsString());
}
- long nextTxId = dir.fsImage.getLastAppliedTxId() + 1;
+ long nextTxId = getFSImage().getLastAppliedTxId() + 1;
LOG.info("Will take over writing edit logs at txnid " +
nextTxId);
editLog.setNextTxId(nextTxId);
- dir.fsImage.editLog.openForWrite();
+ getFSImage().editLog.openForWrite();
}
// Enable quota checks.
@@ -1155,13 +1159,13 @@ public class FSNamesystem implements Nam
((NameNodeEditLogRoller)nnEditLogRoller.getRunnable()).stop();
nnEditLogRoller.interrupt();
}
- if (dir != null && dir.fsImage != null) {
- if (dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null) {
+ if (getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
// Update the fsimage with the last txid that we wrote
// so that the tailer starts from the right spot.
- dir.fsImage.updateLastAppliedTxIdFromWritten();
+ getFSImage().updateLastAppliedTxIdFromWritten();
}
if (cacheManager != null) {
cacheManager.stopMonitorThread();
@@ -1184,9 +1188,9 @@ public class FSNamesystem implements Nam
*/
void startStandbyServices(final Configuration conf) throws IOException {
LOG.info("Starting services required for standby state");
- if (!dir.fsImage.editLog.isOpenForRead()) {
+ if (!getFSImage().editLog.isOpenForRead()) {
// During startup, we're already open for read.
- dir.fsImage.editLog.initSharedJournalsForRead();
+ getFSImage().editLog.initSharedJournalsForRead();
}
blockManager.setPostponeBlocksFromFuture(true);
@@ -1233,8 +1237,8 @@ public class FSNamesystem implements Nam
if (editLogTailer != null) {
editLogTailer.stop();
}
- if (dir != null && dir.fsImage != null && dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null && getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
}
@@ -1477,9 +1481,9 @@ public class FSNamesystem implements Nam
* Version of @see #getNamespaceInfo() that is not protected by a lock.
*/
NamespaceInfo unprotectedGetNamespaceInfo() {
- return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(),
+ return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(),
getClusterId(), getBlockPoolId(),
- dir.fsImage.getStorage().getCTime());
+ getFSImage().getStorage().getCTime());
}
/**
@@ -1497,12 +1501,10 @@ public class FSNamesystem implements Nam
try {
stopActiveServices();
stopStandbyServices();
- if (dir != null) {
- dir.close();
- }
} catch (IOException ie) {
- LOG.error("Error closing FSDirectory", ie);
+ } finally {
IOUtils.cleanup(LOG, dir);
+ IOUtils.cleanup(LOG, fsImage);
}
}
}
@@ -4502,7 +4504,7 @@ public class FSNamesystem implements Nam
* @return registration ID
*/
String getRegistrationID() {
- return Storage.getRegistrationID(dir.fsImage.getStorage());
+ return Storage.getRegistrationID(getFSImage().getStorage());
}
/**
@@ -4718,7 +4720,7 @@ public class FSNamesystem implements Nam
}
public FSImage getFSImage() {
- return dir.fsImage;
+ return fsImage;
}
public FSEditLog getEditLog() {
@@ -7043,7 +7045,7 @@ public class FSNamesystem implements Nam
@Override // NameNodeMXBean
public String getClusterId() {
- return dir.fsImage.getStorage().getClusterID();
+ return getFSImage().getStorage().getClusterID();
}
@Override // NameNodeMXBean
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri Jun 20 18:56:00 2014
@@ -830,7 +830,7 @@ public class NameNode implements NameNod
/** get FSImage */
@VisibleForTesting
public FSImage getFSImage() {
- return namesystem.dir.fsImage;
+ return namesystem.getFSImage();
}
/**
@@ -1140,7 +1140,7 @@ public class NameNode implements NameNod
return true;
}
}
- nsys.dir.fsImage.doRollback(nsys);
+ nsys.getFSImage().doRollback(nsys);
return false;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
Fri Jun 20 18:56:00 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.AccessControlException;
import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
/**
* This is a utility class to expose NameNode functionality for unit tests.
@@ -177,8 +178,9 @@ public class NameNodeAdapter {
}
public static FSImage spyOnFsImage(NameNode nn1) {
- FSImage spy = Mockito.spy(nn1.getNamesystem().dir.fsImage);
- nn1.getNamesystem().dir.fsImage = spy;
+ FSNamesystem fsn = nn1.getNamesystem();
+ FSImage spy = Mockito.spy(fsn.getFSImage());
+ Whitebox.setInternalState(fsn, "fsImage", spy);
return spy;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
Fri Jun 20 18:56:00 2014
@@ -75,8 +75,7 @@ public class TestFSPermissionChecker {
return new PermissionStatus(SUPERUSER, SUPERGROUP, perm);
}
}).when(fsn).createFsOwnerPermissions(any(FsPermission.class));
- FSImage image = mock(FSImage.class);
- dir = new FSDirectory(image, fsn, conf);
+ dir = new FSDirectory(fsn, conf);
inodeRoot = dir.getRoot();
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1604244&r1=1604243&r2=1604244&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
Fri Jun 20 18:56:00 2014
@@ -61,6 +61,7 @@ import org.apache.hadoop.test.GenericTes
import org.apache.log4j.Level;
import org.junit.Test;
import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -124,14 +125,14 @@ public class TestSaveNamespace {
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
// Replace the FSImage with a spy
- FSImage originalImage = fsn.dir.fsImage;
+ FSImage originalImage = fsn.getFSImage();
NNStorage storage = originalImage.getStorage();
NNStorage spyStorage = spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = spy(originalImage);
- fsn.dir.fsImage = spyImage;
+ Whitebox.setInternalState(fsn, "fsImage", spyImage);
boolean shouldFail = false; // should we expect the save operation to fail
// inject fault
@@ -233,11 +234,11 @@ public class TestSaveNamespace {
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
// Replace the FSImage with a spy
- FSImage originalImage = fsn.dir.fsImage;
+ FSImage originalImage = fsn.getFSImage();
NNStorage storage = originalImage.getStorage();
FSImage spyImage = spy(originalImage);
- fsn.dir.fsImage = spyImage;
+ Whitebox.setInternalState(fsn, "fsImage", spyImage);
FileSystem fs = FileSystem.getLocal(conf);
File rootDir = storage.getStorageDir(0).getRoot();
@@ -367,14 +368,15 @@ public class TestSaveNamespace {
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
// Replace the FSImage with a spy
- final FSImage originalImage = fsn.dir.fsImage;
+ final FSImage originalImage = fsn.getFSImage();
NNStorage storage = originalImage.getStorage();
storage.close(); // unlock any directories that FSNamesystem's
initialization may have locked
NNStorage spyStorage = spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = spy(originalImage);
- fsn.dir.fsImage = spyImage;
+ Whitebox.setInternalState(fsn, "fsImage", spyImage);
+
spyImage.storage.setStorageDirectories(
FSNamesystem.getNamespaceDirs(conf),
FSNamesystem.getNamespaceEditsDirs(conf));
@@ -532,7 +534,7 @@ public class TestSaveNamespace {
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
// Replace the FSImage with a spy
- final FSImage image = fsn.dir.fsImage;
+ final FSImage image = fsn.getFSImage();
NNStorage storage = image.getStorage();
storage.close(); // unlock any directories that FSNamesystem's
initialization may have locked
storage.setStorageDirectories(
