[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16368093#comment-16368093 ] stack commented on HBASE-20013: --- .001 Throw a DoNotRetryException if Master is stopped instead of a MasterNotRunningException.. The latter means retries. Let me see how this does. Hard to make a test for a behavior so deep. > TestZKPermissionWatcher is flakey > - > > Key: HBASE-20013 > URL: https://issues.apache.org/jira/browse/HBASE-20013 > Project: HBase > Issue Type: Sub-task > Components: flakey >Reporter: stack >Priority: Major > Attachments: HBASE-20013.branch-2.001.patch > > > The last two nightlies failed on this test in here on shutdown: > {code} > 2018-02-16 20:49:35,132 DEBUG [M:0;881c50037eea:35808] > master.MasterRpcServices(1153): Checking to see if procedure is done pid=7 > 2018-02-16 20:49:35,133 DEBUG [M:0;881c50037eea:35808] > client.RpcRetryingCallerImpl(132): Call exception, tries=7, retries=7, > started=8122 ms ago, cancelled=false, msg=null, details=, > exception=org.apache.hadoop.hbase.MasterNotRunningException > at > org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:2736) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getProcedureResult(MasterRpcServices.java:1155) > at > org.apache.hadoop.hbase.client.ShortCircuitMasterConnection.getProcedureResult(ShortCircuitMasterConnection.java:423) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3490) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3487) > at > org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:100) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3055) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3047) > at org.apache.hadoop.hbase.client.HBaseAdmin.access$700(HBaseAdmin.java:224) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(HBaseAdmin.java:3486) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(HBaseAdmin.java:3438) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(HBaseAdmin.java:3394) > at org.apache.hadoop.hbase.client.HBaseAdmin.get(HBaseAdmin.java:2123) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:612) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:586) > at > org.apache.hadoop.hbase.security.access.AccessController.createACLTable(AccessController.java:1130) > at > org.apache.hadoop.hbase.security.access.AccessController.postStartMaster(AccessController.java:1107) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:994) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:540) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:614) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost.postStartMaster(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:969) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2026) > at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:555) > at java.lang.Thread.run(Thread.java:748) > {code} > We get stuck retrying. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16368095#comment-16368095 ] stack commented on HBASE-20013: --- Let me try hadoopqa first > TestZKPermissionWatcher is flakey > - > > Key: HBASE-20013 > URL: https://issues.apache.org/jira/browse/HBASE-20013 > Project: HBase > Issue Type: Sub-task > Components: flakey >Reporter: stack >Assignee: stack >Priority: Major > Fix For: 2.0.0-beta-2 > > Attachments: HBASE-20013.branch-2.001.patch > > > The last two nightlies failed on this test in here on shutdown: > {code} > 2018-02-16 20:49:35,132 DEBUG [M:0;881c50037eea:35808] > master.MasterRpcServices(1153): Checking to see if procedure is done pid=7 > 2018-02-16 20:49:35,133 DEBUG [M:0;881c50037eea:35808] > client.RpcRetryingCallerImpl(132): Call exception, tries=7, retries=7, > started=8122 ms ago, cancelled=false, msg=null, details=, > exception=org.apache.hadoop.hbase.MasterNotRunningException > at > org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:2736) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getProcedureResult(MasterRpcServices.java:1155) > at > org.apache.hadoop.hbase.client.ShortCircuitMasterConnection.getProcedureResult(ShortCircuitMasterConnection.java:423) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3490) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3487) > at > org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:100) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3055) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3047) > at org.apache.hadoop.hbase.client.HBaseAdmin.access$700(HBaseAdmin.java:224) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(HBaseAdmin.java:3486) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(HBaseAdmin.java:3438) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(HBaseAdmin.java:3394) > at org.apache.hadoop.hbase.client.HBaseAdmin.get(HBaseAdmin.java:2123) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:612) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:586) > at > org.apache.hadoop.hbase.security.access.AccessController.createACLTable(AccessController.java:1130) > at > org.apache.hadoop.hbase.security.access.AccessController.postStartMaster(AccessController.java:1107) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:994) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:540) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:614) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost.postStartMaster(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:969) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2026) > at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:555) > at java.lang.Thread.run(Thread.java:748) > {code} > We get stuck retrying. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16368120#comment-16368120 ] Hadoop QA commented on HBASE-20013: --- | (x) *{color:red}-1 overall{color}* | \\ \\ || Vote || Subsystem || Runtime || Comment || | {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m 15s{color} | {color:blue} Docker mode activated. {color} | || || || || {color:brown} Prechecks {color} || | {color:blue}0{color} | {color:blue} findbugs {color} | {color:blue} 0m 0s{color} | {color:blue} Findbugs executables are not available. {color} | | {color:green}+1{color} | {color:green} hbaseanti {color} | {color:green} 0m 0s{color} | {color:green} Patch does not have any anti-patterns. {color} | | {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m 0s{color} | {color:green} The patch does not contain any @author tags. {color} | | {color:red}-1{color} | {color:red} test4tests {color} | {color:red} 0m 0s{color} | {color:red} The patch doesn't appear to include any new or modified tests. Please justify why no new tests are needed for this patch. Also please list what manual steps were performed to verify this patch. {color} | || || || || {color:brown} branch-2 Compile Tests {color} || | {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 3m 17s{color} | {color:green} branch-2 passed {color} | | {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m 41s{color} | {color:green} branch-2 passed {color} | | {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 1m 6s{color} | {color:green} branch-2 passed {color} | | {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 5m 3s{color} | {color:green} branch has no errors when building our shaded downstream artifacts. {color} | | {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m 27s{color} | {color:green} branch-2 passed {color} | || || || || {color:brown} Patch Compile Tests {color} || | {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 3m 22s{color} | {color:green} the patch passed {color} | | {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m 42s{color} | {color:green} the patch passed {color} | | {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m 42s{color} | {color:green} the patch passed {color} | | {color:red}-1{color} | {color:red} checkstyle {color} | {color:red} 1m 6s{color} | {color:red} hbase-server: The patch generated 1 new + 173 unchanged - 1 fixed = 174 total (was 174) {color} | | {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m 0s{color} | {color:green} The patch has no whitespace issues. {color} | | {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 3m 58s{color} | {color:green} patch has no errors when building our shaded downstream artifacts. {color} | | {color:green}+1{color} | {color:green} hadoopcheck {color} | {color:green} 14m 38s{color} | {color:green} Patch does not cause any errors with Hadoop 2.6.5 2.7.4 or 3.0.0. {color} | | {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m 28s{color} | {color:green} the patch passed {color} | || || || || {color:brown} Other Tests {color} || | {color:green}+1{color} | {color:green} unit {color} | {color:green}103m 59s{color} | {color:green} hbase-server in the patch passed. {color} | | {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m 20s{color} | {color:green} The patch does not generate ASF License warnings. {color} | | {color:black}{color} | {color:black} {color} | {color:black}134m 31s{color} | {color:black} {color} | \\ \\ || Subsystem || Report/Notes || | Docker | Client=17.05.0-ce Server=17.05.0-ce Image:yetus/hbase:9f2f2db | | JIRA Issue | HBASE-20013 | | JIRA Patch URL | https://issues.apache.org/jira/secure/attachment/12910994/HBASE-20013.branch-2.001.patch | | Optional Tests | asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile | | uname | Linux f993c698a8c6 3.13.0-139-generic #188-Ubuntu SMP Tue Jan 9 14:43:09 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /home/jenkins/jenkins-slave/workspace/PreCommit-HBASE-Build/component/dev-support/hbase-personality.sh | | git revision | branch-2 / 34d5f2b70e | | maven | version: Apache Maven 3.5.2 (138edd61fd100ec658bfa2d307c43b76940a5d7d; 2017-10-18T07:58:13Z) | | Default Java | 1.8.0_151 | | checkstyle | https://builds.apache.org/job/PreCommit-HBASE-Build/11556/artifact/patchprocess/diff-checkstyle-hbase-server.txt | | Test Results | https://builds.apache.org/job/PreCommit-HBASE-Build/11556/testReport/ | | Max. process+thread count | 5126 (vs. ulimit of 1) | | modules | C: hbase-server U: hbase-server | | Console
[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16368164#comment-16368164 ] Hudson commented on HBASE-20013: FAILURE: Integrated in Jenkins build HBase-Trunk_matrix #4599 (See [https://builds.apache.org/job/HBase-Trunk_matrix/4599/]) HBASE-20013 TestZKPermissionWatcher is flakey (stack: rev 68d509bc1ff7a3bf69a596aed49f238b42ee0679) * (edit) hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java > TestZKPermissionWatcher is flakey > - > > Key: HBASE-20013 > URL: https://issues.apache.org/jira/browse/HBASE-20013 > Project: HBase > Issue Type: Sub-task > Components: flakey >Reporter: stack >Assignee: stack >Priority: Major > Fix For: 2.0.0-beta-2 > > Attachments: HBASE-20013.branch-2.001.patch > > > The last two nightlies failed on this test in here on shutdown: > {code} > 2018-02-16 20:49:35,132 DEBUG [M:0;881c50037eea:35808] > master.MasterRpcServices(1153): Checking to see if procedure is done pid=7 > 2018-02-16 20:49:35,133 DEBUG [M:0;881c50037eea:35808] > client.RpcRetryingCallerImpl(132): Call exception, tries=7, retries=7, > started=8122 ms ago, cancelled=false, msg=null, details=, > exception=org.apache.hadoop.hbase.MasterNotRunningException > at > org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:2736) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getProcedureResult(MasterRpcServices.java:1155) > at > org.apache.hadoop.hbase.client.ShortCircuitMasterConnection.getProcedureResult(ShortCircuitMasterConnection.java:423) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3490) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3487) > at > org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:100) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3055) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3047) > at org.apache.hadoop.hbase.client.HBaseAdmin.access$700(HBaseAdmin.java:224) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(HBaseAdmin.java:3486) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(HBaseAdmin.java:3438) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(HBaseAdmin.java:3394) > at org.apache.hadoop.hbase.client.HBaseAdmin.get(HBaseAdmin.java:2123) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:612) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:586) > at > org.apache.hadoop.hbase.security.access.AccessController.createACLTable(AccessController.java:1130) > at > org.apache.hadoop.hbase.security.access.AccessController.postStartMaster(AccessController.java:1107) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:994) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:540) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:614) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost.postStartMaster(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:969) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2026) > at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:555) > at java.lang.Thread.run(Thread.java:748) > {code} > We get stuck retrying. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16370124#comment-16370124 ] Chia-Ping Tsai commented on HBASE-20013: It is still flaky in branch-1 and branch-1.4. Should we reopen this? Or file another Jira to trace it for branch-1? > TestZKPermissionWatcher is flakey > - > > Key: HBASE-20013 > URL: https://issues.apache.org/jira/browse/HBASE-20013 > Project: HBase > Issue Type: Sub-task > Components: flakey >Reporter: stack >Assignee: stack >Priority: Major > Fix For: 2.0.0-beta-2 > > Attachments: HBASE-20013.branch-2.001.patch > > > The last two nightlies failed on this test in here on shutdown: > {code} > 2018-02-16 20:49:35,132 DEBUG [M:0;881c50037eea:35808] > master.MasterRpcServices(1153): Checking to see if procedure is done pid=7 > 2018-02-16 20:49:35,133 DEBUG [M:0;881c50037eea:35808] > client.RpcRetryingCallerImpl(132): Call exception, tries=7, retries=7, > started=8122 ms ago, cancelled=false, msg=null, details=, > exception=org.apache.hadoop.hbase.MasterNotRunningException > at > org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:2736) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getProcedureResult(MasterRpcServices.java:1155) > at > org.apache.hadoop.hbase.client.ShortCircuitMasterConnection.getProcedureResult(ShortCircuitMasterConnection.java:423) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3490) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3487) > at > org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:100) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3055) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3047) > at org.apache.hadoop.hbase.client.HBaseAdmin.access$700(HBaseAdmin.java:224) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(HBaseAdmin.java:3486) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(HBaseAdmin.java:3438) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(HBaseAdmin.java:3394) > at org.apache.hadoop.hbase.client.HBaseAdmin.get(HBaseAdmin.java:2123) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:612) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:586) > at > org.apache.hadoop.hbase.security.access.AccessController.createACLTable(AccessController.java:1130) > at > org.apache.hadoop.hbase.security.access.AccessController.postStartMaster(AccessController.java:1107) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:994) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:540) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:614) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost.postStartMaster(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:969) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2026) > at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:555) > at java.lang.Thread.run(Thread.java:748) > {code} > We get stuck retrying. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-20013) TestZKPermissionWatcher is flakey
[ https://issues.apache.org/jira/browse/HBASE-20013?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16370140#comment-16370140 ] stack commented on HBASE-20013: --- [~chia7712] I opened HBASE-20025 for backport (Trying to keep the branch-2 JIRA number count down as I delude myself into thinking hbase2 is almost done -- smile). Thanks sir. > TestZKPermissionWatcher is flakey > - > > Key: HBASE-20013 > URL: https://issues.apache.org/jira/browse/HBASE-20013 > Project: HBase > Issue Type: Sub-task > Components: flakey >Reporter: stack >Assignee: stack >Priority: Major > Fix For: 2.0.0-beta-2 > > Attachments: HBASE-20013.branch-2.001.patch > > > The last two nightlies failed on this test in here on shutdown: > {code} > 2018-02-16 20:49:35,132 DEBUG [M:0;881c50037eea:35808] > master.MasterRpcServices(1153): Checking to see if procedure is done pid=7 > 2018-02-16 20:49:35,133 DEBUG [M:0;881c50037eea:35808] > client.RpcRetryingCallerImpl(132): Call exception, tries=7, retries=7, > started=8122 ms ago, cancelled=false, msg=null, details=, > exception=org.apache.hadoop.hbase.MasterNotRunningException > at > org.apache.hadoop.hbase.master.HMaster.checkInitialized(HMaster.java:2736) > at > org.apache.hadoop.hbase.master.MasterRpcServices.getProcedureResult(MasterRpcServices.java:1155) > at > org.apache.hadoop.hbase.client.ShortCircuitMasterConnection.getProcedureResult(ShortCircuitMasterConnection.java:423) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3490) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture$2.rpcCall(HBaseAdmin.java:3487) > at > org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:100) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:107) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3055) > at > org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3047) > at org.apache.hadoop.hbase.client.HBaseAdmin.access$700(HBaseAdmin.java:224) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.getProcedureResult(HBaseAdmin.java:3486) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.waitProcedureResult(HBaseAdmin.java:3438) > at > org.apache.hadoop.hbase.client.HBaseAdmin$ProcedureFuture.get(HBaseAdmin.java:3394) > at org.apache.hadoop.hbase.client.HBaseAdmin.get(HBaseAdmin.java:2123) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:612) > at > org.apache.hadoop.hbase.client.HBaseAdmin.createTable(HBaseAdmin.java:586) > at > org.apache.hadoop.hbase.security.access.AccessController.createACLTable(AccessController.java:1130) > at > org.apache.hadoop.hbase.security.access.AccessController.postStartMaster(AccessController.java:1107) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:994) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost$71.call(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:540) > at > org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:614) > at > org.apache.hadoop.hbase.master.MasterCoprocessorHost.postStartMaster(MasterCoprocessorHost.java:991) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:969) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2026) > at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:555) > at java.lang.Thread.run(Thread.java:748) > {code} > We get stuck retrying. -- This message was sent by Atlassian JIRA (v7.6.3#76005)