Author: szetszwo Date: Fri Jan 10 06:07:24 2014 New Revision: 1557039 URL: http://svn.apache.org/r1557039 Log: Merge r1555021 through r1557038 from trunk.
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1556551-1557038 Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jan 10 06:07:24 2014 @@ -245,6 +245,9 @@ Trunk (Unreleased) HDFS-5715. Use Snapshot ID to indicate the corresponding Snapshot for a FileDiff/DirectoryDiff. (jing9) + HDFS-5721. sharedEditsImage in Namenode#initializeSharedEdits() should be + closed before method returns. (Ted Yu via junping_du) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) @@ -736,6 +739,8 @@ Release 2.4.0 - UNRELEASED HDFS-5690. DataNode fails to start in secure mode when dfs.http.policy equals to HTTP_ONLY. (Haohui Mai via jing9) + HDFS-5449. WebHdfs compatibility broken between 2.2 and 1.x / 23.x (kihwal) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1556551-1557038 Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Jan 10 06:07:24 2014 @@ -622,8 +622,14 @@ public class FSNamesystem implements Nam long loadStart = now(); String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); - namesystem.loadFSImage(startOpt, fsImage, - HAUtil.isHAEnabled(conf, nameserviceId)); + try { + namesystem.loadFSImage(startOpt, fsImage, + HAUtil.isHAEnabled(conf, nameserviceId)); + } catch (IOException ioe) { + LOG.warn("Encountered exception loading fsimage", ioe); + fsImage.close(); + throw ioe; + } long timeTakenToLoadFSImage = now() - loadStart; LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs"); NameNodeMetrics nnMetrics = NameNode.getNameNodeMetrics(); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Jan 10 06:07:24 2014 @@ -816,14 +816,20 @@ public class NameNode implements NameNod System.out.println("Formatting using clusterid: " + clusterId); FSImage fsImage = new FSImage(conf, nameDirsToFormat, editDirsToFormat); - FSNamesystem fsn = new FSNamesystem(conf, fsImage); - fsImage.getEditLog().initJournalsForWrite(); - - if (!fsImage.confirmFormat(force, isInteractive)) { - return true; // aborted + try { + FSNamesystem fsn = new FSNamesystem(conf, fsImage); + fsImage.getEditLog().initJournalsForWrite(); + + if (!fsImage.confirmFormat(force, isInteractive)) { + return true; // aborted + } + + fsImage.format(fsn, clusterId); + } catch (IOException ioe) { + LOG.warn("Encountered exception during format: ", ioe); + fsImage.close(); + throw ioe; } - - fsImage.format(fsn, clusterId); return false; } @@ -897,6 +903,7 @@ public class NameNode implements NameNod } NNStorage existingStorage = null; + FSImage sharedEditsImage = null; try { FSNamesystem fsns = FSNamesystem.loadFromDisk(getConfigurationWithoutSharedEdits(conf)); @@ -906,7 +913,7 @@ public class NameNode implements NameNod List<URI> sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf); - FSImage sharedEditsImage = new FSImage(conf, + sharedEditsImage = new FSImage(conf, Lists.<URI>newArrayList(), sharedEditsDirs); sharedEditsImage.getEditLog().initJournalsForWrite(); @@ -934,6 +941,13 @@ public class NameNode implements NameNod LOG.error("Could not initialize shared edits dir", ioe); return true; // aborted } finally { + if (sharedEditsImage != null) { + try { + sharedEditsImage.close(); + } catch (IOException ioe) { + LOG.warn("Could not close sharedEditsImage", ioe); + } + } // Have to unlock storage explicitly for the case when we're running in a // unit test, which runs in the same JVM as NNs. if (existingStorage != null) { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java Fri Jan 10 06:07:24 2014 @@ -190,24 +190,29 @@ public class BootstrapStandby implements // Load the newly formatted image, using all of the directories (including shared // edits) FSImage image = new FSImage(conf); - image.getStorage().setStorageInfo(storage); - image.initEditLog(); - assert image.getEditLog().isOpenForRead() : + try { + image.getStorage().setStorageInfo(storage); + image.initEditLog(); + assert image.getEditLog().isOpenForRead() : "Expected edit log to be open for read"; - - // Ensure that we have enough edits already in the shared directory to - // start up from the last checkpoint on the active. - if (!checkLogsAvailableForRead(image, imageTxId, curTxId)) { - return ERR_CODE_LOGS_UNAVAILABLE; - } - - image.getStorage().writeTransactionIdFileToStorage(curTxId); - // Download that checkpoint into our storage directories. - MD5Hash hash = TransferFsImage.downloadImageToStorage( + // Ensure that we have enough edits already in the shared directory to + // start up from the last checkpoint on the active. + if (!checkLogsAvailableForRead(image, imageTxId, curTxId)) { + return ERR_CODE_LOGS_UNAVAILABLE; + } + + image.getStorage().writeTransactionIdFileToStorage(curTxId); + + // Download that checkpoint into our storage directories. + MD5Hash hash = TransferFsImage.downloadImageToStorage( otherHttpAddr, imageTxId, storage, true); - image.saveDigestAndRenameCheckpointImage(imageTxId, hash); + image.saveDigestAndRenameCheckpointImage(imageTxId, hash); + } catch (IOException ioe) { + image.close(); + throw ioe; + } return 0; } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Fri Jan 10 06:07:24 2014 @@ -271,7 +271,7 @@ public class JsonUtil { } /** Convert a DatanodeInfo to a Json map. */ - private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) { + static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) { if (datanodeinfo == null) { return null; } @@ -279,6 +279,9 @@ public class JsonUtil { // TODO: Fix storageID final Map<String, Object> m = new TreeMap<String, Object>(); m.put("ipAddr", datanodeinfo.getIpAddr()); + // 'name' is equivalent to ipAddr:xferPort. Older clients (1.x, 0.23.x) + // expects this instead of the two fields. + m.put("name", datanodeinfo.getXferAddr()); m.put("hostName", datanodeinfo.getHostName()); m.put("storageID", datanodeinfo.getDatanodeUuid()); m.put("xferPort", datanodeinfo.getXferPort()); @@ -325,17 +328,49 @@ public class JsonUtil { } /** Convert a Json map to an DatanodeInfo object. */ - static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) { + static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) + throws IOException { if (m == null) { return null; } + // ipAddr and xferPort are the critical fields for accessing data. + // If any one of the two is missing, an exception needs to be thrown. + + // Handle the case of old servers (1.x, 0.23.x) sending 'name' instead + // of ipAddr and xferPort. + String ipAddr = getString(m, "ipAddr", null); + int xferPort = getInt(m, "xferPort", -1); + if (ipAddr == null) { + String name = getString(m, "name", null); + if (name != null) { + int colonIdx = name.indexOf(':'); + if (colonIdx > 0) { + ipAddr = name.substring(0, colonIdx); + xferPort = Integer.parseInt(name.substring(colonIdx +1)); + } else { + throw new IOException( + "Invalid value in server response: name=[" + name + "]"); + } + } else { + throw new IOException( + "Missing both 'ipAddr' and 'name' in server response."); + } + // ipAddr is non-null & non-empty string at this point. + } + + // Check the validity of xferPort. + if (xferPort == -1) { + throw new IOException( + "Invalid or missing 'xferPort' in server response."); + } + // TODO: Fix storageID return new DatanodeInfo( - (String)m.get("ipAddr"), + ipAddr, (String)m.get("hostName"), (String)m.get("storageID"), - (int)(long)(Long)m.get("xferPort"), + xferPort, (int)(long)(Long)m.get("infoPort"), getInt(m, "infoSecurePort", 0), (int)(long)(Long)m.get("ipcPort"), @@ -368,7 +403,8 @@ public class JsonUtil { } /** Convert an Object[] to a DatanodeInfo[]. */ - private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) { + private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) + throws IOException { if (objects == null) { return null; } else if (objects.length == 0) { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java?rev=1557039&r1=1557038&r2=1557039&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Fri Jan 10 06:07:24 2014 @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.util.Time; @@ -61,7 +62,7 @@ public class TestJsonUtil { } @Test - public void testToDatanodeInfoWithoutSecurePort() { + public void testToDatanodeInfoWithoutSecurePort() throws Exception { Map<String, Object> response = new HashMap<String, Object>(); response.put("ipAddr", "127.0.0.1"); @@ -84,4 +85,63 @@ public class TestJsonUtil { JsonUtil.toDatanodeInfo(response); } + + @Test + public void testToDatanodeInfoWithName() throws Exception { + Map<String, Object> response = new HashMap<String, Object>(); + + // Older servers (1.x, 0.23, etc.) sends 'name' instead of ipAddr + // and xferPort. + String name = "127.0.0.1:1004"; + response.put("name", name); + response.put("hostName", "localhost"); + response.put("storageID", "fake-id"); + response.put("infoPort", 1338l); + response.put("ipcPort", 1339l); + response.put("capacity", 1024l); + response.put("dfsUsed", 512l); + response.put("remaining", 512l); + response.put("blockPoolUsed", 512l); + response.put("lastUpdate", 0l); + response.put("xceiverCount", 4096l); + response.put("networkLocation", "foo.bar.baz"); + response.put("adminState", "NORMAL"); + response.put("cacheCapacity", 123l); + response.put("cacheUsed", 321l); + + DatanodeInfo di = JsonUtil.toDatanodeInfo(response); + Assert.assertEquals(name, di.getXferAddr()); + + // The encoded result should contain name, ipAddr and xferPort. + Map<String, Object> r = JsonUtil.toJsonMap(di); + Assert.assertEquals(name, (String)r.get("name")); + Assert.assertEquals("127.0.0.1", (String)r.get("ipAddr")); + // In this test, it is Integer instead of Long since json was not actually + // involved in constructing the map. + Assert.assertEquals(1004, (int)(Integer)r.get("xferPort")); + + // Invalid names + String[] badNames = {"127.0.0.1", "127.0.0.1:", ":", "127.0.0.1:sweet", ":123"}; + for (String badName : badNames) { + response.put("name", badName); + checkDecodeFailure(response); + } + + // Missing both name and ipAddr + response.remove("name"); + checkDecodeFailure(response); + + // Only missing xferPort + response.put("ipAddr", "127.0.0.1"); + checkDecodeFailure(response); + } + + private void checkDecodeFailure(Map<String, Object> map) { + try { + JsonUtil.toDatanodeInfo(map); + Assert.fail("Exception not thrown against bad input."); + } catch (Exception e) { + // expected + } + } }