Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."
This reverts commit e61491d4769536ce8b77b416005ce531278ce729. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f4817f2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f4817f2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f4817f2 Branch: refs/heads/YARN-5734 Commit: 8f4817f2c5524c1891a3b3b50e396dd4ca294a2d Parents: e61491d Author: Andrew Wang <w...@apache.org> Authored: Thu Mar 2 16:20:48 2017 -0800 Committer: Andrew Wang <w...@apache.org> Committed: Thu Mar 2 16:20:48 2017 -0800 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 7 +---- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++- .../hdfs/server/datanode/FileIoProvider.java | 2 +- .../server/datanode/ProfilingFileIoEvents.java | 27 +++++--------------- .../datanode/TestDataNodeVolumeMetrics.java | 4 +-- .../hadoop/tools/TestHdfsConfigFields.java | 2 ++ 6 files changed, 17 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index a8bdbeb..7900692 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -332,12 +332,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i FsVolume -------- -Per-volume metrics contain Datanode Volume IO related statistics. Per-volume -metrics are off by default. They can be enabled by setting `dfs.datanode -.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0. -Setting this value to 0.0 would mean profiling is not enabled. But enabling -per-volume metrics may have a performance impact. Each metrics record -contains tags such as Hostname as additional information along with metrics. +Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics. | Name | Description | |:---- |:---- | http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 82d6073..be20829 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -718,6 +718,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling // Datanode File IO Stats + public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY = + "dfs.datanode.enable.fileio.profiling"; + public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT = + false; public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY = "dfs.datanode.enable.fileio.fault.injection"; public static final boolean @@ -726,7 +730,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY = "dfs.datanode.fileio.profiling.sampling.fraction"; public static final double - DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0; + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0; //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java index 5508e0b..9def2e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java @@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*; * * Behavior can be injected into these events by enabling the * profiling and/or fault injection event hooks through - * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}. * These event hooks are disabled by default. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index 7eab969..9af84fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; -import static java.lang.Math.abs; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -44,32 +43,18 @@ class ProfilingFileIoEvents { public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { + isEnabled = conf.getBoolean(DFSConfigKeys + .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys + .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT); - if (abs(fileIOSamplingFraction) < 0.000001) { - LOG.info(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " - + fileIOSamplingFraction + ". Disabling file IO profiling"); - isEnabled = false; - } else if (fileIOSamplingFraction < 0.000001) { + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT); + if (fileIOSamplingFraction > 1) { LOG.warn(DFSConfigKeys .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + - " value cannot be less than 0. Disabling file IO profiling."); - isEnabled = false; - } else if (fileIOSamplingFraction > 1) { - LOG.warn(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + - " value cannot be more than 1. Setting value to 1 and enabling " + - "file IO profiling"); - isEnabled = true; + " value cannot be more than 1. Setting value to 1"); fileIOSamplingFraction = 1; - } else { - LOG.info(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " - + fileIOSamplingFraction + ". Enabling file IO profiling"); - isEnabled = true; } sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java index 03e1dee..6a8ac9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java @@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics { private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setDouble(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); + conf.setBoolean(DFSConfigKeys + .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true); SimulatedFSDataset.setFactory(conf); return new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_DATANODES) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index 1fdf713..cdce342 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase { .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY); configurationPropsToSkipCompare .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY); + configurationPropsToSkipCompare + .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
