[ https://issues.apache.org/jira/browse/MYRIAD-171?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14994663#comment-14994663 ]
Brandon Gulla commented on MYRIAD-171: -------------------------------------- 2.7.0, Absolutely. The defined hadoop version in the docker build is 2.7.1 currently but I can that patch tonight. I think taking our time with the reflection and setting up a ShimLoader will benefit us greater in the long run. > Hadoop-2.7.1 UpdateFIle > ----------------------- > > Key: MYRIAD-171 > URL: https://issues.apache.org/jira/browse/MYRIAD-171 > Project: Myriad > Issue Type: Bug > Affects Versions: Myriad 0.1.0 > Reporter: Brandon Gulla > Assignee: Brandon Gulla > Priority: Blocker > > Hadoop 2.7.1 introduced an API changed that breaks Myriad when using HA > ResourceManager. They added an extra argument to the method that we are > currently using. > To get around this, switch hadoop version to 2.7.0 or wait for the impending > code change. > Hadoop 2.7.1 > https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java > Hadoop 2.7.0 > https://github.com/apache/hadoop/blob/release-2.7.0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java > I1106 19:26:55.098060 5135 sched.cpp:164] Version: 0.25.0 > 15/11/06 19:26:55 INFO scheduler.MyriadDriver: Driver started with status: > DRIVER_RUNNING > 15/11/06 19:26:55 INFO scheduler.MyriadDriverManager: Driver started with > status: DRIVER_RUNNING > 15/11/06 19:26:55 INFO myriad.Main: started mesosDriver.. > 15/11/06 19:26:55 INFO interceptor.CompositeInterceptor: Registered > org.apache.myriad.policy.LeastAMNodesFirstPolicy into the registry. > I1106 19:26:55.100406 5185 sched.cpp:262] New master detected at > master@172.31.45.229:5050 > I1106 19:26:55.100591 5185 sched.cpp:272] No credentials provided. > Attempting to register without authentication > 15/11/06 19:26:55 INFO myriad.Main: Launching 1 NM(s) with profile medium > I1106 19:26:55.102533 5179 sched.cpp:641] Framework registered with > 2f8adca0-12da-4200-867d-2b742955cabd-0004 > 15/11/06 19:26:55 INFO scheduler.MyriadOperations: Adding 1 NM instances to > cluster > 15/11/06 19:26:55 INFO handlers.RegisteredEventHandler: Received event: > org.apache.myriad.scheduler.event.RegisteredEvent@6a005de5 with frameworkId: > value: "2f8adca0-12da-4200-867d-2b742955cabd-0004" > Nov 06, 2015 7:26:55 PM com.lmax.disruptor.FatalExceptionHandler > handleEventException > SEVERE: Exception processing: 0 > org.apache.myriad.scheduler.event.RegisteredEvent@6a005de5 > java.lang.NoSuchMethodError: > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.updateFile(Lorg/apache/hadoop/fs/Path;[B)V > at > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.storeMyriadState(MyriadFileSystemRMStateStore.java:95) > at > org.apache.myriad.state.SchedulerState.updateStateStore(SchedulerState.java:374) > at > org.apache.myriad.state.SchedulerState.setFrameworkId(SchedulerState.java:362) > at > org.apache.myriad.scheduler.event.handlers.RegisteredEventHandler.onEvent(RegisteredEventHandler.java:42) > at > org.apache.myriad.scheduler.event.handlers.RegisteredEventHandler.onEvent(RegisteredEventHandler.java:30) > at > com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:128) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > 15/11/06 19:26:55 INFO handlers.ResourceOffersEventHandler: Received offers 3 > 15/11/06 19:26:55 ERROR yarn.YarnUncaughtExceptionHandler: Thread > Thread[pool-6-thread-1,5,main] threw an Exception. > java.lang.RuntimeException: java.lang.NoSuchMethodError: > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.updateFile(Lorg/apache/hadoop/fs/Path;[B)V > at > com.lmax.disruptor.FatalExceptionHandler.handleEventException(FatalExceptionHandler.java:45) > at > com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:147) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.NoSuchMethodError: > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.updateFile(Lorg/apache/hadoop/fs/Path;[B)V > at > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.storeMyriadState(MyriadFileSystemRMStateStore.java:95) > at > org.apache.myriad.state.SchedulerState.updateStateStore(SchedulerState.java:374) > at > org.apache.myriad.state.SchedulerState.setFrameworkId(SchedulerState.java:362) > at > org.apache.myriad.scheduler.event.handlers.RegisteredEventHandler.onEvent(RegisteredEventHandler.java:42) > at > org.apache.myriad.scheduler.event.handlers.RegisteredEventHandler.onEvent(RegisteredEventHandler.java:30) > at > com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:128) > ... 3 more > 15/11/06 19:26:55 FATAL resourcemanager.ResourceManager: Error starting > ResourceManager > java.lang.NoSuchMethodError: > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.updateFile(Lorg/apache/hadoop/fs/Path;[B)V > at > org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore.storeMyriadState(MyriadFileSystemRMStateStore.java:95) > at > org.apache.myriad.state.SchedulerState.updateStateStore(SchedulerState.java:374) > at > org.apache.myriad.state.SchedulerState.addTask(SchedulerState.java:87) > at > org.apache.myriad.state.SchedulerState.addNodes(SchedulerState.java:75) > at > org.apache.myriad.scheduler.MyriadOperations.flexUpCluster(MyriadOperations.java:77) > at org.apache.myriad.Main.startNMInstances(Main.java:227) > at org.apache.myriad.Main.run(Main.java:113) > at org.apache.myriad.Main.initialize(Main.java:88) > at > org.apache.myriad.scheduler.yarn.interceptor.MyriadInitializationInterceptor.init(MyriadInitializationInterceptor.java:49) > at > org.apache.myriad.scheduler.yarn.interceptor.CompositeInterceptor.init(CompositeInterceptor.java:92) > at > org.apache.myriad.scheduler.yarn.MyriadFairScheduler.serviceStart(MyriadFairScheduler.java:75) > at > org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) > at > org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:584) > at > org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:994) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1035) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1031) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:422) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1031) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1071) > at > org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) > at > org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.main(ResourceManager.java:1208) -- This message was sent by Atlassian JIRA (v6.3.4#6332)