[ https://issues.apache.org/jira/browse/HBASE-25929?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yi Mei resolved HBASE-25929. ---------------------------- Fix Version/s: 2.4.4 2.3.6 2.5.0 3.0.0-alpha-1 Resolution: Fixed > RegionServer JVM crash when compaction > -------------------------------------- > > Key: HBASE-25929 > URL: https://issues.apache.org/jira/browse/HBASE-25929 > Project: HBase > Issue Type: Bug > Components: Compaction > Affects Versions: 3.0.0-alpha-1, 2.5.0, 2.3.5, 2.4.3 > Reporter: Yi Mei > Assignee: Yi Mei > Priority: Critical > Fix For: 3.0.0-alpha-1, 2.5.0, 2.3.6, 2.4.4 > > Attachments: hs_err_pid27712.log, hs_err_pid28814.log > > > In our cluster, we found region servers may be crashed in several cases. > In hs_err_pid27712.log: > {code:java} > Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) > J 2687 sun.misc.Unsafe.copyMemory(Ljava/lang/Object;JLjava/lang/Object;JJ)V > (0 bytes) @ 0x00007f85c987eda7 [0x00007f85c987ed40+0x67] > J 5884 C1 > org.apache.hadoop.hbase.util.UnsafeAccess.unsafeCopy(Ljava/lang/Object;JLjava/lang/Object;JJ)V > (62 bytes) @ 0x00007f85c93fd904 [0x00007f85c93fd780+0x184] > J 4274 C1 > org.apache.hadoop.hbase.util.UnsafeAccess.copy(Ljava/nio/ByteBuffer;I[BII)V > (73 bytes) @ 0x00007f85c9d57a94 [0x00007f85c9d574a0+0x5f4] > J 5211 C2 > org.apache.hadoop.hbase.util.ByteBufferUtils.copyFromBufferToArray([BLjava/nio/ByteBuffer;III)V > (69 bytes) @ 0x00007f85ca039a34 [0x00007f85ca0399a0+0x94] > J 5985 C1 > org.apache.hadoop.hbase.CellUtil.copyQualifierTo(Lorg/apache/hadoop/hbase/Cell;[BI)I > (59 bytes) @ 0x00007f85c9296a34 [0x00007f85c92964c0+0x574] > J 6011 C1 org.apache.hadoop.hbase.ByteBufferKeyValue.getQualifierArray()[B (5 > bytes) @ 0x00007f85c913e094 [0x00007f85c913d4c0+0xbd4] > J 6004 C1 > org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(Lorg/apache/hadoop/hbase/Cell;Ljava/util/function/Function;)Ljava/lang/String; > (211 bytes) @ 0x00007f85c93737b4 [0x00007f85c93722e0+0x14d4] > J 6000 C1 > org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(Lorg/apache/hadoop/hbase/Cell;)Ljava/lang/String; > (10 bytes) @ 0x00007f85c9854d14 [0x00007f85c9854ba0+0x174] > j > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.getMidpoint(Lorg/apache/hadoop/hbase/CellComparator;Lorg/apache/hadoop/hbase/Cell;Lorg/apache/hadoop/hbase/Cell;Lorg/apache/hadoop/hbase/io/hfile/HFileContext;)Lorg/apache/hadoop/hbase/Cell;+132 > j org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishBlock()V+102 > j org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.checkBlockBoundary()V+32 > j > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.append(Lorg/apache/hadoop/hbase/Cell;)V+77 > j > org.apache.hadoop.hbase.regionserver.StoreFileWriter.append(Lorg/apache/hadoop/hbase/Cell;)V+20 > j > org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$FileDetails;Lorg/apache/hadoop/hbase/regionserver/InternalScanner;Lorg/apache/hadoop/hbase/regionserver/CellSink;JZLorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;ZI)Z+318 > j > org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$InternalScannerFactory;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$CellSinkFactory;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+221 > j > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+12 > j > org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+16 > j > org.apache.hadoop.hbase.regionserver.HStore.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+194 > {code} > In hs_err_pid28814.log: > {code:java} > Stack: [0x00007f6d8e69b000,0x00007f6d8e6dc000], sp=0x00007f6d8e6d9e88, free > space=251k > Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native > code) > V [libjvm.so+0x747fa0] > J 2989 sun.misc.Unsafe.copyMemory(Ljava/lang/Object;JLjava/lang/Object;JJ)V > (0 bytes) @ 0x00007f751db756e1 [0x00007f751db75600+0xe1] > j > org.apache.hadoop.hbase.util.UnsafeAccess.unsafeCopy(Ljava/lang/Object;JLjava/lang/Object;JJ)V+36 > j > org.apache.hadoop.hbase.util.UnsafeAccess.copy(Ljava/nio/ByteBuffer;I[BII)V+69 > j > org.apache.hadoop.hbase.util.ByteBufferUtils.copyFromBufferToArray([BLjava/nio/ByteBuffer;III)V+39 > j > org.apache.hadoop.hbase.CellUtil.copyQualifierTo(Lorg/apache/hadoop/hbase/Cell;[BI)I+31 > J 12082 C2 org.apache.hadoop.hbase.ByteBufferKeyValue.getQualifierArray()[B > (5 bytes) @ 0x00007f751ef15fbc [0x00007f751ef15dc0+0x1fc] > J 16584 C2 > org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(Lorg/apache/hadoop/hbase/Cell;Ljava/util/function/Function;)Ljava/lang/String; > (211 bytes) @ 0x00007f751fe320b8 [0x00007f751fe31b80+0x538] > J 17007 C2 > org.apache.hadoop.hbase.regionserver.StoreFileWriter.append(Lorg/apache/hadoop/hbase/Cell;)V > (31 bytes) @ 0x00007f751fc2c0f4 [0x00007f751fc2aac0+0x1634] > J 17178 C2 > org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$FileDetails;Lorg/apache/hadoop/hbase/regionserver/InternalScanner;Lorg/apache/hadoop/hbase/regionserver/CellSink;JZLorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;ZI)Z > (767 bytes) @ 0x00007f751f8e330c [0x00007f751f8e2960+0x9ac] > j > org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$InternalScannerFactory;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$CellSinkFactory;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+221 > j > org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+12 > j > org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+16 > j > org.apache.hadoop.hbase.regionserver.HStore.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List;+194 > {code} > Sometimes, RS is not crashed but we can see the following logs: > {code:java} > 2021-05-27T12:53:54,465 ERROR > [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769-shortCompactions-2] > regionserver.CompactSplit$CompactionRunner(640): Compaction failed > Request=regionName=t1,user00000000000000000080,1622091224198.cf38ea5f2ea0d90163b53c2b2fd329d2., > storeName=A, fileCount=2, fileSize=2.0 M (1009.7 K, 1009.7 K), priority=1, > time=16220912334172021-05-27T12:53:54,465 ERROR > [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769-shortCompactions-2] > regionserver.CompactSplit$CompactionRunner(640): Compaction failed > Request=regionName=t1,user00000000000000000080,1622091224198.cf38ea5f2ea0d90163b53c2b2fd329d2., > storeName=A, fileCount=2, fileSize=2.0 M (1009.7 K, 1009.7 K), priority=1, > time=1622091233417java.lang.IllegalArgumentException: Left byte array sorts > after right row; left=user00000000000000000080, > right=user00000000000000000001 at > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.getMinimumMidpointArray(HFileWriterImpl.java:445) > ~[classes/:?] at > org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.getMidpoint(HFileWriterImpl.java:390) > ~[classes/:?] > {code} > Because byte buffers of cells have been already released, but RS still need > to use these cells. -- This message was sent by Atlassian Jira (v8.3.4#803005)