Author: szetszwo
Date: Sat Mar 1 01:14:38 2014
New Revision: 1573120
URL: http://svn.apache.org/r1573120
Log:
Merge r1569890 through r1573119 from trunk.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1573043-1573119
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Sat Mar 1 01:14:38 2014
@@ -109,9 +109,6 @@ Trunk (Unreleased)
HDFS-4052. BlockManager#invalidateWork should print log outside the lock.
(Jing Zhao via suresh)
- HDFS-4200. Reduce the size of synchronized sections in PacketResponder.
- (suresh)
-
HDFS-3358. Specify explicitly that the NN UI status total is talking
of persistent objects on heap. (harsh)
@@ -370,6 +367,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6030. Remove an unused constructor in INode.java. (yzhang via
cmccabe)
+ HDFS-4200. Reduce the size of synchronized sections in PacketResponder.
+ (suresh)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
@@ -505,6 +505,15 @@ Release 2.4.0 - UNRELEASED
HDFS-5339. WebHDFS URI does not accept logical nameservices when security
is
enabled. (Haohui Mai via jing9)
+ HDFS-6033. PBImageXmlWriter incorrectly handles processing cache
+ directives. (atm)
+
+ HDFS-5821. TestHDFSCLI fails for user names with the dash character.
+ (Gera Shegalov via Arpit Agarwal)
+
+ HDFS-5956. A file size is multiplied by the replication factor in 'hdfs oiv
+ -p FileDistribution' option. (Akira Ajisaka via wheat9)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Propchange:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1573043-1573119
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
Sat Mar 1 01:14:38 2014
@@ -75,6 +75,9 @@ import org.apache.hadoop.security.protoc
import
org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
import
org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
+import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
+import
org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolClientSideTranslatorPB;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import
org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
@@ -252,13 +255,16 @@ public class NameNodeProxies {
} else if (xface == RefreshAuthorizationPolicyProtocol.class) {
proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
conf, ugi);
+ } else if (xface == RefreshCallQueueProtocol.class) {
+ proxy = (T) createNNProxyWithRefreshCallQueueProtocol(nnAddr, conf, ugi);
} else {
- String message = "Upsupported protocol found when creating the proxy " +
+ String message = "Unsupported protocol found when creating the proxy " +
"connection to NameNode: " +
((xface != null) ? xface.getClass().getName() : "null");
LOG.error(message);
throw new IllegalStateException(message);
}
+
return new ProxyAndInfo<T>(proxy, dtService);
}
@@ -286,6 +292,14 @@ public class NameNodeProxies {
return new RefreshUserMappingsProtocolClientSideTranslatorPB(proxy);
}
+ private static RefreshCallQueueProtocol
+ createNNProxyWithRefreshCallQueueProtocol(InetSocketAddress address,
+ Configuration conf, UserGroupInformation ugi) throws IOException {
+ RefreshCallQueueProtocolPB proxy = (RefreshCallQueueProtocolPB)
+ createNameNodeProxy(address, conf, ugi,
RefreshCallQueueProtocolPB.class, 0);
+ return new RefreshCallQueueProtocolClientSideTranslatorPB(proxy);
+ }
+
private static GetUserMappingsProtocol
createNNProxyWithGetUserMappingsProtocol(
InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
throws IOException {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Sat Mar 1 01:14:38 2014
@@ -58,6 +58,7 @@ import org.apache.hadoop.security.Refres
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -230,6 +231,8 @@ public class NameNode implements NameNod
return RefreshAuthorizationPolicyProtocol.versionID;
} else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
return RefreshUserMappingsProtocol.versionID;
+ } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) {
+ return RefreshCallQueueProtocol.versionID;
} else if (protocol.equals(GetUserMappingsProtocol.class.getName())){
return GetUserMappingsProtocol.versionID;
} else {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
Sat Mar 1 01:14:38 2014
@@ -141,6 +141,9 @@ import org.apache.hadoop.security.protoc
import
org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
import
org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
+import
org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolServerSideTranslatorPB;
+import
org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueProtocolService;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import
org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetUserMappingsProtocolService;
@@ -218,6 +221,11 @@ class NameNodeRpcServer implements Namen
BlockingService refreshUserMappingService =
RefreshUserMappingsProtocolService
.newReflectiveBlockingService(refreshUserMappingXlator);
+ RefreshCallQueueProtocolServerSideTranslatorPB refreshCallQueueXlator =
+ new RefreshCallQueueProtocolServerSideTranslatorPB(this);
+ BlockingService refreshCallQueueService = RefreshCallQueueProtocolService
+ .newReflectiveBlockingService(refreshCallQueueXlator);
+
GetUserMappingsProtocolServerSideTranslatorPB getUserMappingXlator =
new GetUserMappingsProtocolServerSideTranslatorPB(this);
BlockingService getUserMappingService = GetUserMappingsProtocolService
@@ -264,6 +272,9 @@ class NameNodeRpcServer implements Namen
refreshAuthService, serviceRpcServer);
DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class,
refreshUserMappingService, serviceRpcServer);
+ // We support Refreshing call queue here in case the client RPC queue is
full
+ DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class,
+ refreshCallQueueService, serviceRpcServer);
DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
getUserMappingService, serviceRpcServer);
@@ -306,6 +317,8 @@ class NameNodeRpcServer implements Namen
refreshAuthService, clientRpcServer);
DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class,
refreshUserMappingService, clientRpcServer);
+ DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class,
+ refreshCallQueueService, clientRpcServer);
DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
getUserMappingService, clientRpcServer);
@@ -1128,6 +1141,17 @@ class NameNodeRpcServer implements Namen
ProxyUsers.refreshSuperUserGroupsConfiguration();
}
+
+ @Override // RefreshCallQueueProtocol
+ public void refreshCallQueue() {
+ LOG.info("Refreshing call queue.");
+
+ Configuration conf = new Configuration();
+ clientRpcServer.refreshCallQueue(conf);
+ if (this.serviceRpcServer != null) {
+ serviceRpcServer.refreshCallQueue(conf);
+ }
+ }
@Override // GetUserMappingsProtocol
public String[] getGroupsForUser(String user) throws IOException {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java
Sat Mar 1 01:14:38 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
/** The full set of RPC methods implemented by the Namenode. */
@@ -33,6 +34,7 @@ public interface NamenodeProtocols
NamenodeProtocol,
RefreshAuthorizationPolicyProtocol,
RefreshUserMappingsProtocol,
+ RefreshCallQueueProtocol,
GetUserMappingsProtocol,
HAServiceProtocol {
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
Sat Mar 1 01:14:38 2014
@@ -66,6 +66,7 @@ import org.apache.hadoop.security.Refres
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -652,6 +653,7 @@ public class DFSAdmin extends FsShell {
"\t[-refreshServiceAcl]\n" +
"\t[-refreshUserToGroupsMappings]\n" +
"\t[-refreshSuperUserGroupsConfiguration]\n" +
+ "\t[-refreshCallQueue]\n" +
"\t[-printTopology]\n" +
"\t[-refreshNamenodes datanodehost:port]\n"+
"\t[-deleteBlockPool datanodehost:port blockpoolId [force]]\n"+
@@ -723,6 +725,8 @@ public class DFSAdmin extends FsShell {
String refreshSuperUserGroupsConfiguration =
"-refreshSuperUserGroupsConfiguration: Refresh superuser proxy groups
mappings\n";
+ String refreshCallQueue = "-refreshCallQueue: Reload the call queue from
config\n";
+
String printTopology = "-printTopology: Print a tree of the racks and
their\n" +
"\t\tnodes as reported by the Namenode\n";
@@ -805,6 +809,8 @@ public class DFSAdmin extends FsShell {
System.out.println(refreshUserToGroupsMappings);
} else if ("refreshSuperUserGroupsConfiguration".equals(cmd)) {
System.out.println(refreshSuperUserGroupsConfiguration);
+ } else if ("refreshCallQueue".equals(cmd)) {
+ System.out.println(refreshCallQueue);
} else if ("printTopology".equals(cmd)) {
System.out.println(printTopology);
} else if ("refreshNamenodes".equals(cmd)) {
@@ -843,6 +849,7 @@ public class DFSAdmin extends FsShell {
System.out.println(refreshServiceAcl);
System.out.println(refreshUserToGroupsMappings);
System.out.println(refreshSuperUserGroupsConfiguration);
+ System.out.println(refreshCallQueue);
System.out.println(printTopology);
System.out.println(refreshNamenodes);
System.out.println(deleteBlockPool);
@@ -1034,6 +1041,27 @@ public class DFSAdmin extends FsShell {
return 0;
}
+ public int refreshCallQueue() throws IOException {
+ // Get the current configuration
+ Configuration conf = getConf();
+
+ // for security authorization
+ // server principal for this call
+ // should be NN's one.
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
+ conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
+
+ // Create the client
+ RefreshCallQueueProtocol refreshProtocol =
+ NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
+ RefreshCallQueueProtocol.class).getProxy();
+
+ // Refresh the user-to-groups mappings
+ refreshProtocol.refreshCallQueue();
+
+ return 0;
+ }
+
/**
* Displays format of commands.
* @param cmd The command that is being executed.
@@ -1093,6 +1121,9 @@ public class DFSAdmin extends FsShell {
} else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-refreshSuperUserGroupsConfiguration]");
+ } else if ("-refreshCallQueue".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-refreshCallQueue]");
} else if ("-printTopology".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-printTopology]");
@@ -1125,6 +1156,7 @@ public class DFSAdmin extends FsShell {
System.err.println(" [-refreshServiceAcl]");
System.err.println(" [-refreshUserToGroupsMappings]");
System.err.println(" [-refreshSuperUserGroupsConfiguration]");
+ System.err.println(" [-refreshCallQueue]");
System.err.println(" [-printTopology]");
System.err.println(" [-refreshNamenodes datanodehost:port]");
System.err.println(" [-deleteBlockPool datanode-host:port
blockpoolId [force]]");
@@ -1315,6 +1347,8 @@ public class DFSAdmin extends FsShell {
exitCode = refreshUserToGroupsMappings();
} else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
exitCode = refreshSuperUserGroupsConfiguration();
+ } else if ("-refreshCallQueue".equals(cmd)) {
+ exitCode = refreshCallQueue();
} else if ("-printTopology".equals(cmd)) {
exitCode = printTopology();
} else if ("-refreshNamenodes".equals(cmd)) {
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
Sat Mar 1 01:14:38 2014
@@ -123,10 +123,10 @@ final class FileDistributionCalculator {
totalBlocks += f.getBlocksCount();
long fileSize = 0;
for (BlockProto b : f.getBlocksList()) {
- fileSize += b.getNumBytes() * f.getReplication();
+ fileSize += b.getNumBytes();
}
maxFileSize = Math.max(fileSize, maxFileSize);
- totalSpace += fileSize;
+ totalSpace += fileSize * f.getReplication();
int bucket = fileSize > maxSize ? distribution.length - 1 : (int) Math
.ceil((double)fileSize / steps);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
Sat Mar 1 01:14:38 2014
@@ -157,7 +157,7 @@ public final class PBImageXmlWriter {
.o("maxRelativeExpiry", p.getMaxRelativeExpiry());
out.print("</pool>\n");
}
- for (int i = 0; i < s.getNumPools(); ++i) {
+ for (int i = 0; i < s.getNumDirectives(); ++i) {
CacheDirectiveInfoProto p = CacheDirectiveInfoProto
.parseDelimitedFrom(is);
out.print("<directive>");
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
Sat Mar 1 01:14:38 2014
@@ -39,6 +39,8 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import
org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
import
org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
+import
org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import
org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
import org.junit.AfterClass;
@@ -169,4 +171,15 @@ public class TestIsMethodSupported {
assertTrue(
translator.isMethodSupported("refreshUserToGroupsMappings"));
}
+
+ @Test
+ public void testRefreshCallQueueProtocol() throws IOException {
+ RefreshCallQueueProtocolClientSideTranslatorPB translator =
+ (RefreshCallQueueProtocolClientSideTranslatorPB)
+ NameNodeProxies.createNonHAProxy(conf, nnAddress,
+ RefreshCallQueueProtocol.class,
+ UserGroupInformation.getCurrentUser(), true).getProxy();
+ assertTrue(
+ translator.isMethodSupported("refreshCallQueue"));
+ }
}
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1573120&r1=1573119&r2=1573120&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
Sat Mar 1 01:14:38 2014
@@ -28,6 +28,8 @@ import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.io.StringReader;
import java.io.StringWriter;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -241,7 +243,7 @@ public class TestOfflineImageViewer {
}
@Test
- public void testFileDistributionVisitor() throws IOException {
+ public void testFileDistributionCalculator() throws IOException {
StringWriter output = new StringWriter();
PrintWriter o = new PrintWriter(output);
new FileDistributionCalculator(new Configuration(), 0, 0, o)
@@ -250,10 +252,29 @@ public class TestOfflineImageViewer {
Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
Matcher matcher = p.matcher(output.getBuffer());
-
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1));
- assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
+ assertEquals(NUM_DIRS * FILES_PER_DIR, totalFiles);
+
+ p = Pattern.compile("totalDirectories = (\\d+)\n");
+ matcher = p.matcher(output.getBuffer());
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ int totalDirs = Integer.parseInt(matcher.group(1));
+ // totalDirs includes root directory
+ assertEquals(NUM_DIRS + 1, totalDirs);
+
+ FileStatus maxFile = Collections.max(writtenFiles.values(),
+ new Comparator<FileStatus>() {
+ @Override
+ public int compare(FileStatus first, FileStatus second) {
+ return first.getLen() < second.getLen() ? -1 :
+ ((first.getLen() == second.getLen()) ? 0 : 1);
+ }
+ });
+ p = Pattern.compile("maxFileSize = (\\d+)\n");
+ matcher = p.matcher(output.getBuffer());
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
}
@Test
