Repository: hadoop Updated Branches: refs/heads/trunk ab3885f2c -> ba12f87dc
YARN-8390. Fix API incompatible changes in FairScheduler's AllocationFileLoaderService. (Gergo Repas via Haibo Chen) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba12f87d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba12f87d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba12f87d Branch: refs/heads/trunk Commit: ba12f87dcb0e406da57cdd1ad17677ac2367f425 Parents: ab3885f Author: Haibo Chen <haiboc...@apache.org> Authored: Mon Jun 4 09:14:53 2018 -0700 Committer: Haibo Chen <haiboc...@apache.org> Committed: Mon Jun 4 09:15:23 2018 -0700 ---------------------------------------------------------------------- .../fair/AllocationFileLoaderService.java | 16 ++-- .../scheduler/fair/FairScheduler.java | 4 +- .../fair/TestAllocationFileLoaderService.java | 96 ++++++++++---------- 3 files changed, 58 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba12f87d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 7a40b6a..e541ab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -87,7 +87,7 @@ public class AllocationFileLoaderService extends AbstractService { private Path allocFile; private FileSystem fs; - private final Listener reloadListener; + private Listener reloadListener; @VisibleForTesting long reloadIntervalMs = ALLOC_RELOAD_INTERVAL_MS; @@ -95,16 +95,15 @@ public class AllocationFileLoaderService extends AbstractService { private Thread reloadThread; private volatile boolean running = true; - public AllocationFileLoaderService(Listener reloadListener) { - this(reloadListener, SystemClock.getInstance()); + public AllocationFileLoaderService() { + this(SystemClock.getInstance()); } private List<Permission> defaultPermissions; - public AllocationFileLoaderService(Listener reloadListener, Clock clock) { + public AllocationFileLoaderService(Clock clock) { super(AllocationFileLoaderService.class.getName()); this.clock = clock; - this.reloadListener = reloadListener; } @Override @@ -209,6 +208,10 @@ public class AllocationFileLoaderService extends AbstractService { return allocPath; } + public synchronized void setReloadListener(Listener reloadListener) { + this.reloadListener = reloadListener; + } + /** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. @@ -350,6 +353,7 @@ public class AllocationFileLoaderService extends AbstractService { public interface Listener { void onReload(AllocationConfiguration info) throws IOException; - void onCheck(); + default void onCheck() { + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba12f87d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4c84aa9..123f711 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -208,8 +208,7 @@ public class FairScheduler extends public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); - allocsLoader = - new AllocationFileLoaderService(new AllocationReloadListener()); + allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); } @@ -1438,6 +1437,7 @@ public class FairScheduler extends } allocsLoader.init(conf); + allocsLoader.setReloadListener(new AllocationReloadListener()); // If we fail to load allocations file on initialize, we want to fail // immediately. After a successful load, exceptions on future reloads // will just result in leaving things as they are. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba12f87d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 30b8a91..50a003e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.Listener; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; @@ -33,8 +32,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Fai import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Test; -import org.mockito.Mockito; - import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; @@ -82,8 +79,7 @@ public class TestAllocationFileLoaderService { fs.copyFromLocalFile(new Path(fschedURL.toURI()), new Path(fsAllocPath)); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, fsAllocPath); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(Mockito.mock(Listener.class)); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(fsAllocPath, allocationFile.toString()); assertTrue(fs.exists(allocationFile)); @@ -96,8 +92,7 @@ public class TestAllocationFileLoaderService { throws UnsupportedFileSystemException { Configuration conf = new YarnConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, "badfs:///badfile"); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(Mockito.mock(Listener.class)); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.getAllocationFile(conf); } @@ -110,7 +105,7 @@ public class TestAllocationFileLoaderService { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, TEST_FAIRSCHED_XML); AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(Mockito.mock(Listener.class)); + new AllocationFileLoaderService(); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(TEST_FAIRSCHED_XML, allocationFile.getName()); assertTrue(fs.exists(allocationFile)); @@ -139,11 +134,12 @@ public class TestAllocationFileLoaderService { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder, clock); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService( + clock); allocLoader.reloadIntervalMs = 5; allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; @@ -209,9 +205,7 @@ public class TestAllocationFileLoaderService { public void testAllocationFileParsing() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); AllocationFileWriter .create() @@ -284,6 +278,8 @@ public class TestAllocationFileLoaderService { .writeToFile(ALLOC_FILE); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; @@ -431,9 +427,7 @@ public class TestAllocationFileLoaderService { public void testBackwardsCompatibleAllocationFileParsing() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); @@ -479,6 +473,8 @@ public class TestAllocationFileLoaderService { out.close(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; @@ -554,10 +550,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; @@ -588,10 +584,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } @@ -612,10 +608,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } @@ -636,10 +632,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } @@ -658,10 +654,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); try { allocLoader.reloadAllocations(); } catch (AllocationConfigurationException ex) { @@ -689,10 +685,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); try { allocLoader.reloadAllocations(); } catch (AllocationConfigurationException ex) { @@ -718,10 +714,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; // Check whether queue 'parent' and 'child' are loaded successfully @@ -749,10 +745,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } @@ -771,10 +767,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } @@ -797,10 +793,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; @@ -857,10 +853,10 @@ public class TestAllocationFileLoaderService { out.println("</allocations>"); out.close(); - ReloadListener confHolder = new ReloadListener(); - AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(confHolder); + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
