Repository: hadoop Updated Branches: refs/heads/HDFS-10285 e5bd57145 -> 16af3f2d6 (forced update)
HDFS-11264: [SPS]: Double checks to ensure that SPS/Mover are not running together. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f27e270 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f27e270 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f27e270 Branch: refs/heads/HDFS-10285 Commit: 3f27e27070b5142ab801df64136f0ef056f0c153 Parents: 1a093b6 Author: Uma Maheswara Rao G <uma.ganguma...@intel.com> Authored: Wed Jul 12 17:56:56 2017 -0700 Committer: Uma Maheswara Rao Gangumalla <umamah...@apache.org> Committed: Thu Aug 9 20:47:20 2018 -0700 ---------------------------------------------------------------------- .../server/namenode/StoragePolicySatisfier.java | 53 +++++++++++--------- .../namenode/TestStoragePolicySatisfier.java | 3 +- ...stStoragePolicySatisfierWithStripedFile.java | 5 +- 3 files changed, 34 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f27e270/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java index 97cbf1b..00b4cd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java @@ -128,6 +128,14 @@ public class StoragePolicySatisfier implements Runnable { */ public synchronized void start(boolean reconfigStart) { isRunning = true; + if (checkIfMoverRunning()) { + isRunning = false; + LOG.error( + "Stopping StoragePolicySatisfier thread " + "as Mover ID file " + + HdfsServerConstants.MOVER_ID_PATH.toString() + + " been opened. Maybe a Mover instance is running!"); + return; + } if (reconfigStart) { LOG.info("Starting StoragePolicySatisfier, as admin requested to " + "activate it."); @@ -211,20 +219,6 @@ public class StoragePolicySatisfier implements Runnable { @Override public void run() { - boolean isMoverRunning = !checkIfMoverRunning(); - synchronized (this) { - isRunning = isMoverRunning; - if (!isRunning) { - // Stopping monitor thread and clearing queues as well - this.clearQueues(); - this.storageMovementsMonitor.stopGracefully(); - LOG.error( - "Stopping StoragePolicySatisfier thread " + "as Mover ID file " - + HdfsServerConstants.MOVER_ID_PATH.toString() - + " been opened. Maybe a Mover instance is running!"); - return; - } - } while (namesystem.isRunning() && isRunning) { try { if (!namesystem.isInSafeMode()) { @@ -274,25 +268,34 @@ public class StoragePolicySatisfier implements Runnable { // we want to check block movements. Thread.sleep(3000); } catch (Throwable t) { - synchronized (this) { + handleException(t); + } + } + } + + private void handleException(Throwable t) { + // double check to avoid entering into synchronized block. + if (isRunning) { + synchronized (this) { + if (isRunning) { isRunning = false; // Stopping monitor thread and clearing queues as well this.clearQueues(); this.storageMovementsMonitor.stopGracefully(); - } - if (!namesystem.isRunning()) { - LOG.info("Stopping StoragePolicySatisfier."); - if (!(t instanceof InterruptedException)) { - LOG.info("StoragePolicySatisfier received an exception" - + " while shutting down.", t); + if (!namesystem.isRunning()) { + LOG.info("Stopping StoragePolicySatisfier."); + if (!(t instanceof InterruptedException)) { + LOG.info("StoragePolicySatisfier received an exception" + + " while shutting down.", t); + } + return; } - break; } - LOG.error("StoragePolicySatisfier thread received runtime exception. " - + "Stopping Storage policy satisfier work", t); - break; } } + LOG.error("StoragePolicySatisfier thread received runtime exception. " + + "Stopping Storage policy satisfier work", t); + return; } private BlocksMovingAnalysisStatus analyseBlocksStorageMovementsAndAssignToDN( http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f27e270/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java index 7127895..be7236b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java @@ -927,7 +927,8 @@ public class TestStoragePolicySatisfier { String fooDir = "/foo"; client.mkdirs(fooDir, new FsPermission((short) 777), true); // set an EC policy on "/foo" directory - client.setErasureCodingPolicy(fooDir, null); + client.setErasureCodingPolicy(fooDir, + StripedFileTestUtil.getDefaultECPolicy().getName()); // write file to fooDir final String testFile = "/foo/bar"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f27e270/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java index 195c9e3..f905ead 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java @@ -323,6 +323,8 @@ public class TestStoragePolicySatisfierWithStripedFile { conf.set(DFSConfigKeys .DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_KEY, "3000"); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); initConfWithStripe(conf, defaultStripeBlockSize); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) @@ -346,7 +348,8 @@ public class TestStoragePolicySatisfierWithStripedFile { Path barDir = new Path("/bar"); fs.mkdirs(barDir); // set an EC policy on "/bar" directory - fs.setErasureCodingPolicy(barDir, null); + fs.setErasureCodingPolicy(barDir, + StripedFileTestUtil.getDefaultECPolicy().getName()); // write file to barDir final Path fooFile = new Path("/bar/foo"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
