Chandra Sekhar K created HBASE-29007:
----------------------------------------
Summary: Master Failed to Start after upgrading from 2.5.10 to
3.0.0-beta-1
Key: HBASE-29007
URL: https://issues.apache.org/jira/browse/HBASE-29007
Project: HBase
Issue Type: Bug
Reporter: Chandra Sekhar K
Assignee: Chandra Sekhar K
Attachments: hbase-hdoop-master-master.log,
hbase-hdoop-regionserver-master.log
Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1
{code:java}
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
master.HMaster: Failed to become active master
java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl
[FAILED] to be RUNNING, but the service has FAILED
at
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384)
~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
at
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324)
~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
at
org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155)
~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException:
Failed 18 actions: RetriesExhaustedException: 18 times, servers with issues:
at
org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.makeError(BufferedMutatorOverAsyncBufferedMutator.java:107)
~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.internalFlush(BufferedMutatorOverAsyncBufferedMutator.java:122)
~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.close(BufferedMutatorOverAsyncBufferedMutator.java:166)
~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.TableNamespaceManager.migrateNamespaceTable(TableNamespaceManager.java:92)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:122)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:61)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:252)
~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
at
org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1532)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
... 5 more
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
master.HMaster: Master server abort: loaded coprocessors are: []
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster]
master.HMaster: ***** ABORTING master master,16000,1733059831148: Unhandled
exception. Starting shutdown. *****
java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl
[FAILED] to be RUNNING, but the service has FAILED
at
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384)
~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
at
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324)
~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
at
org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155)
~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
at java.lang.Thread.run(Thread.java:840) ~[?:?]{code}
Region Server has below warning log
{code:java}
2024-12-01T19:00:47,354 WARN
[RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020]
regionserver.HRegion: No such column family in batch mutation in region
org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation@74bf756d
org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family
ns does not exist in region hbase:meta,,1.1588230740 in table 'hbase:meta',
{TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 =>
'|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|',
METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info',
INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE',
DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0',
REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true',
COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'},
{NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS =>
'2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING =>
'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE =>
'0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE',
BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table',
INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE',
DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0',
REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true',
COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
at
org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5344)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5331)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion.checkFamilies(HRegion.java:5325)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion.access$100(HRegion.java:239)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPreparePut(HRegion.java:3849)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3486)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3498)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation$1.visit(HRegion.java:3878)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.visitBatchOperations(HRegion.java:3377)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPrepare(HRegion.java:3857)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4753)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4673)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:984)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:902)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:867)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2830)
~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45018)
~[hbase-protocol-shaded-3.0.0-beta-1.jar:3.0.0-beta-1]{code}
complete log files are attached
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
