Repository: hadoop Updated Branches: refs/heads/branch-2 a754f94ed -> fcd23b61e
HDFS-7706. Switch BlockManager logging to use slf4j. (cherry picked from commit 951b3608a8cb1d9063b9be9c740b524c137b816f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcd23b61 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcd23b61 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcd23b61 Branch: refs/heads/branch-2 Commit: fcd23b61e30b521d74709297c2808225fa82e906 Parents: a754f94 Author: Andrew Wang <w...@apache.org> Authored: Fri Jan 30 11:32:25 2015 -0800 Committer: Andrew Wang <w...@apache.org> Committed: Fri Jan 30 11:36:02 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../server/blockmanagement/BlockManager.java | 8 +++++--- .../blockmanagement/InvalidateBlocks.java | 4 ++-- .../PendingReplicationBlocks.java | 4 ++-- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 7 +++++++ .../TestPendingInvalidateBlock.java | 4 ++-- .../hdfs/server/namenode/ha/TestDNFencing.java | 11 +++-------- .../namenode/ha/TestPipelinesFailover.java | 11 +++++------ .../namenode/ha/TestStandbyBlockManagement.java | 20 ++------------------ .../server/namenode/ha/TestStandbyIsHot.java | 7 +------ 10 files changed, 31 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index efe7b49..cef4185 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -278,6 +278,8 @@ Release 2.7.0 - UNRELEASED HDFS-7603. The background replication queue initialization may not let others run (kihwal) + HDFS-7706. Switch BlockManager logging to use slf4j. (wang) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index ec085d7..4c745d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -38,7 +38,6 @@ import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -89,6 +88,8 @@ import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Keeps information related to the blocks stored in the Hadoop cluster. @@ -96,7 +97,7 @@ import com.google.common.collect.Sets; @InterfaceAudience.Private public class BlockManager { - static final Log LOG = LogFactory.getLog(BlockManager.class); + static final Logger LOG = LoggerFactory.getLogger(BlockManager.class); public static final Log blockLog = NameNode.blockStateChangeLog; private static final String QUEUE_REASON_CORRUPT_STATE = @@ -3622,7 +3623,8 @@ public class BlockManager { LOG.info("Stopping ReplicationMonitor for testing."); break; } - LOG.fatal("ReplicationMonitor thread received Runtime exception. ", t); + LOG.error("ReplicationMonitor thread received Runtime exception. ", + t); terminate(1, t); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java index 574b5b6..ad29769 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; @@ -37,6 +36,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.hdfs.DFSUtil; import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; /** * Keeps a Collection for every named machine containing blocks @@ -67,7 +67,7 @@ class InvalidateBlocks { printBlockDeletionTime(BlockManager.LOG); } - private void printBlockDeletionTime(final Log log) { + private void printBlockDeletionTime(final Logger log) { log.info(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY + " is set to " + DFSUtil.durationToString(pendingPeriodInMs)); SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java index 2b507e7..57c2962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java @@ -28,9 +28,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.util.Daemon; +import org.slf4j.Logger; /*************************************************** * PendingReplicationBlocks does the bookkeeping of all @@ -44,7 +44,7 @@ import org.apache.hadoop.util.Daemon; * ***************************************************/ class PendingReplicationBlocks { - private static final Log LOG = BlockManager.LOG; + private static final Logger LOG = BlockManager.LOG; private final Map<Block, PendingBlockInfo> pendingReplications; private final ArrayList<Block> timedOutItems; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 9250224..a6a5807 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -81,6 +81,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.VersionInfo; +import org.apache.log4j.Level; import org.junit.Assume; import org.mockito.internal.util.reflection.Whitebox; @@ -1661,4 +1662,10 @@ public class DFSTestUtil { } }, 100, waitTime); } + + public static void setNameNodeLogLevel(Level level) { + GenericTestUtils.setLogLevel(LogFactory.getLog(FSNamesystem.class), level); + GenericTestUtils.setLogLevel(LogFactory.getLog(BlockManager.class), level); + GenericTestUtils.setLogLevel(NameNode.stateChangeLog, level); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index e10d824..84ae771 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.text.SimpleDateFormat; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; @@ -29,6 +28,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; @@ -42,7 +42,7 @@ import org.mockito.internal.util.reflection.Whitebox; */ public class TestPendingInvalidateBlock { { - ((Log4JLogger)BlockManager.LOG).getLogger().setLevel(Level.DEBUG); + GenericTestUtils.setLogLevel(BlockManager.LOG, Level.DEBUG); } private static final int BLOCKSIZE = 1024; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java index 85864f7..f625bb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java @@ -25,9 +25,10 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CountDownLatch; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -50,7 +51,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -65,9 +65,6 @@ import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; -import com.google.common.base.Supplier; -import com.google.common.collect.Lists; - public class TestDNFencing { @@ -82,9 +79,7 @@ public class TestDNFencing { private FileSystem fs; static { - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + DFSTestUtil.setNameNodeLogLevel(Level.ALL); } @Before http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java index bba3dbb..5d816e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java @@ -53,12 +53,14 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.retry.RetryInvocationHandler; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; import org.apache.hadoop.test.MultithreadedTestUtil.RepeatingTestThread; import org.apache.hadoop.test.MultithreadedTestUtil.TestContext; import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.junit.Test; import org.mockito.Mockito; @@ -69,12 +71,9 @@ import com.google.common.base.Supplier; */ public class TestPipelinesFailover { static { - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog( - "org.apache.hadoop.io.retry.RetryInvocationHandler")).getLogger().setLevel(Level.ALL); - - ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + GenericTestUtils.setLogLevel(LogFactory.getLog(RetryInvocationHandler + .class), Level.ALL); + DFSTestUtil.setNameNodeLogLevel(Level.ALL); } protected static final Log LOG = LogFactory.getLog( http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java index d78d43b..9042f8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java @@ -17,13 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -31,22 +26,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; -import org.junit.Assert; import org.junit.Test; -import com.google.common.base.Supplier; +import static org.junit.Assert.assertEquals; /** * Makes sure that standby doesn't do the unnecessary block management such as @@ -60,9 +46,7 @@ public class TestStandbyBlockManagement { private static final Path TEST_FILE_PATH = new Path(TEST_FILE); static { - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + DFSTestUtil.setNameNodeLogLevel(Level.ALL); } @Test(timeout=60000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd23b61/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java index ecd5243..622ed94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -35,10 +34,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.test.GenericTestUtils; @@ -60,9 +57,7 @@ public class TestStandbyIsHot { private static final Path TEST_FILE_PATH = new Path(TEST_FILE); static { - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL); - ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + DFSTestUtil.setNameNodeLogLevel(Level.ALL); } @Test(timeout=60000)
