Repository: hadoop Updated Branches: refs/heads/trunk a0b5a0a41 -> 850d679ac
HADOOP-12447. Clean up some htrace integration issues (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/850d679a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/850d679a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/850d679a Branch: refs/heads/trunk Commit: 850d679acb935a6a6b0e6cb6f69d998e99395468 Parents: a0b5a0a Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Tue Sep 29 09:25:11 2015 -0700 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Tue Sep 29 09:25:11 2015 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Tracing.md | 15 +++++++-------- .../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 -- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 --- .../hdfs/server/namenode/NameNodeRpcServer.java | 2 +- .../hadoop/hdfs/server/namenode/NamenodeFsck.java | 5 ++++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 -------- .../blockmanagement/TestBlockTokenWithDFS.java | 2 ++ .../server/datanode/TestDataNodeVolumeFailure.java | 2 ++ .../org/apache/hadoop/tracing/TestTraceAdmin.java | 8 ++++---- 9 files changed, 20 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md index 7897855..4cc6a07 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md @@ -85,8 +85,8 @@ You need to run the command against all servers if you want to update the config You need to specify the class name of span receiver as argument of `-class` option. You can specify the configuration associated with span receiver by `-Ckey=value` options. - $ hadoop trace -add -class LocalFileSpanReceiver -Cdfs.htrace.local-file-span-receiver.path=/tmp/htrace.out -host 192.168.56.2:9000 - Added trace span receiver 2 with configuration dfs.htrace.local-file-span-receiver.path = /tmp/htrace.out + $ hadoop trace -add -class org.apache.htrace.core.LocalFileSpanReceiver -Chadoop.htrace.local.file.span.receiver.path=/tmp/htrace.out -host 192.168.56.2:9000 + Added trace span receiver 2 with configuration hadoop.htrace.local.file.span.receiver.path = /tmp/htrace.out $ hadoop trace -list -host 192.168.56.2:9000 ID CLASS @@ -137,8 +137,7 @@ which start tracing span before invoking HDFS shell command. FsShell shell = new FsShell(); conf.setQuietMode(false); shell.setConf(conf); - Tracer tracer = new Tracer.Builder(). - name("TracingFsShell). + Tracer tracer = new Tracer.Builder("TracingFsShell"). conf(TraceUtils.wrapHadoopConf("tracing.fs.shell.htrace.", conf)). build(); int res = 0; @@ -177,15 +176,15 @@ ProbabilitySampler. ```xml <property> - <name>dfs.client.htrace.spanreceiver.classes</name> + <name>hadoop.htrace.span.receiver.classes</name> <value>LocalFileSpanReceiver</value> </property> <property> - <name>dfs.client.htrace.sampler</name> + <name>fs.client.htrace.sampler.classes</name> <value>ProbabilitySampler</value> </property> <property> - <name>dfs.client.htrace.sampler.fraction</name> - <value>0.5</value> + <name>fs.client.htrace.sampler.fraction</name> + <value>0.01</value> </property> ``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index fe31531..992c374 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -149,8 +149,6 @@ public interface HdfsClientConfigKeys { "dfs.client.test.drop.namenode.response.number"; int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0; String DFS_CLIENT_LOCAL_INTERFACES = "dfs.client.local.interfaces"; - // HDFS client HTrace configuration. - String DFS_CLIENT_HTRACE_PREFIX = "dfs.client.htrace."; String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix"; String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b631955..055d7e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -64,9 +64,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT = HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT; - public static final String DFS_CLIENT_HTRACE_SAMPLER_CLASSES = - "dfs.client.htrace.sampler.classes"; - // HA related configuration public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration"; public static final long DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT = 50; http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 79a3773..551a0e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -419,7 +419,7 @@ class NameNodeRpcServer implements NamenodeProtocols { UnresolvedPathException.class); clientRpcServer.setTracer(nn.tracer); if (serviceRpcServer != null) { - clientRpcServer.setTracer(nn.tracer); + serviceRpcServer.setTracer(nn.tracer); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 6d40898..309012e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -80,6 +80,7 @@ import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.tracing.TraceUtils; import org.apache.hadoop.util.Time; import org.apache.htrace.core.Tracer; @@ -202,7 +203,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { this.staleInterval = conf.getLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT); - this.tracer = new Tracer.Builder("NamenodeFsck").build(); + this.tracer = new Tracer.Builder("NamenodeFsck"). + conf(TraceUtils.wrapHadoopConf("namenode.fsck.htrace.", conf)). + build(); for (Iterator<String> it = pmap.keySet().iterator(); it.hasNext();) { String key = it.next(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f1b855e..77b556e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2428,14 +2428,6 @@ </property> <property> - <name>dfs.client.htrace.sampler.classes</name> - <value></value> - <description> - The class names of the HTrace Samplers to use for the HDFS client. - </description> -</property> - -<property> <name>dfs.ha.zkfc.nn.http.timeout.ms</name> <value>20000</value> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java index 50d548a..e39ab3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsTracer; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; @@ -161,6 +162,7 @@ public class TestBlockTokenWithDFS { setCachingStrategy(CachingStrategy.newDefaultStrategy()). setClientCacheContext(ClientContext.getFromConf(conf)). setConfiguration(conf). + setTracer(FsTracer.get(conf)). setRemotePeerFactory(new RemotePeerFactory() { @Override public Peer newConnectedPeer(InetSocketAddress addr, http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index cb50edc..2c4fcc5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FsTracer; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; @@ -515,6 +516,7 @@ public class TestDataNodeVolumeFailure { setCachingStrategy(CachingStrategy.newDefaultStrategy()). setClientCacheContext(ClientContext.getFromConf(conf)). setConfiguration(conf). + setTracer(FsTracer.get(conf)). setRemotePeerFactory(new RemotePeerFactory() { @Override public Peer newConnectedPeer(InetSocketAddress addr, http://git-wip-us.apache.org/repos/asf/hadoop/blob/850d679a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java index b08866b..7e10d90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java @@ -73,10 +73,10 @@ public class TestTraceAdmin { Assert.assertEquals("ret:0, [no span receivers found]" + NEWLINE, runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster))); Assert.assertEquals("ret:0, Added trace span receiver 1 with " + - "configuration dfs.htrace.local-file-span-receiver.path = " + tracePath + NEWLINE, + "configuration hadoop.htrace.local.file.span.receiver.path = " + tracePath + NEWLINE, runTraceCommand(trace, "-add", "-host", getHostPortForNN(cluster), "-class", "org.apache.htrace.core.LocalFileSpanReceiver", - "-Cdfs.htrace.local-file-span-receiver.path=" + tracePath)); + "-Chadoop.htrace.local.file.span.receiver.path=" + tracePath)); String list = runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster)); Assert.assertTrue(list.startsWith("ret:0")); @@ -87,10 +87,10 @@ public class TestTraceAdmin { Assert.assertEquals("ret:0, [no span receivers found]" + NEWLINE, runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster))); Assert.assertEquals("ret:0, Added trace span receiver 2 with " + - "configuration dfs.htrace.local-file-span-receiver.path = " + tracePath + NEWLINE, + "configuration hadoop.htrace.local.file.span.receiver.path = " + tracePath + NEWLINE, runTraceCommand(trace, "-add", "-host", getHostPortForNN(cluster), "-class", "LocalFileSpanReceiver", - "-Cdfs.htrace.local-file-span-receiver.path=" + tracePath)); + "-Chadoop.htrace.local.file.span.receiver.path=" + tracePath)); Assert.assertEquals("ret:0, Removed trace span receiver 2" + NEWLINE, runTraceCommand(trace, "-remove", "2", "-host", getHostPortForNN(cluster)));