hbase git commit: HBASE-19930: Fixing the bug, in the rare case when there is a merge into CCM and the one of the old segments has a big cell allocated on-heap and it needs to be copied to the MSLAB
Repository: hbase
Updated Branches:
refs/heads/branch-2 ce5f5e6bc -> af804a586
HBASE-19930: Fixing the bug, in the rare case when there is a merge into CCM
and the one of the old segments has a big cell allocated on-heap and it needs
to be copied to the MSLAB
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af804a58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af804a58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af804a58
Branch: refs/heads/branch-2
Commit: af804a586b39babc60cde3f9a228134d636e9047
Parents: ce5f5e6
Author: anastas
Authored: Thu Feb 15 14:35:41 2018 +0200
Committer: anastas
Committed: Thu Feb 15 14:35:41 2018 +0200
--
.../regionserver/ImmutableMemStoreLAB.java | 13 +++-
.../TestCompactingToCellFlatMapMemStore.java| 66
2 files changed, 78 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/af804a58/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
index 93d2685..871f526 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
@@ -45,9 +45,20 @@ public class ImmutableMemStoreLAB implements MemStoreLAB {
throw new IllegalStateException("This is an Immutable MemStoreLAB.");
}
+ /**
+ * The process of merging assumes all cells are allocated on mslab.
+ * There is a rare case in which the first immutable segment,
+ * participating in a merge, is a CSLM.
+ * Since the CSLM hasn't been flattened yet, and there is no point in
flattening it (since it is
+ * going to be merged), its big cells (for whom size > maxAlloc) must be
copied into mslab.
+ * This method copies the passed cell into the first mslab in the mslabs
list,
+ * returning either a new cell instance over the copied data,
+ * or null when this cell cannt be copied.
+ */
@Override
public Cell forceCopyOfBigCellInto(Cell cell) {
-throw new IllegalStateException("This is an Immutable MemStoreLAB.");
+MemStoreLAB mslab = this.mslabs.get(0);
+return mslab.forceCopyOfBigCellInto(cell);
}
/* Creating chunk to be used as index chunk in CellChunkMap, part of the
chunks array.
http://git-wip-us.apache.org/repos/asf/hbase/blob/af804a58/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 25265b3..106c215 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellComparatorImpl;
@@ -825,6 +826,71 @@ public class TestCompactingToCellFlatMapMemStore extends
TestCompactingMemStore
assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize());
}
+ /**
+ * CellChunkMap Segment index requires all cell data to be written in the
MSLAB Chunks.
+ * Even though MSLAB is enabled, cells bigger than the size of a chunk are
not
+ * written in the MSLAB Chunks.
+ * If such cells are found in the process of a merge they need to be copied
into MSLAB.
+ * testForceCopyOfBigCellIntoImmutableSegment checks that the
+ * ImmutableMemStoreLAB's forceCopyOfBigCellInto does what it's supposed to
do.
+ */
+ @Test
+ public void testForceCopyOfBigCellIntoImmutableSegment() throws IOException {
+
+if (toCellChunkMap == false) {
+ return;
+}
+
+// set memstore to flat into CellChunkMap
+MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC;
+
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
+String.valueOf(compactionType));
+((MyCompactingMemStore) memstore).initiateType(compactionType,
memstore.getConfiguration());
+((CompactingMemStore)
memstore).setIndexTy
hbase git commit: HBASE-19930: Fixing the bug, in the rare case when there is a merge into CCM and the one of the old segments has a big cell allocated on-heap and it needs to be copied to the MSLAB
Repository: hbase
Updated Branches:
refs/heads/master c18e7a963 -> 07e93458f
HBASE-19930: Fixing the bug, in the rare case when there is a merge into CCM
and the one of the old segments has a big cell allocated on-heap and it needs
to be copied to the MSLAB
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/07e93458
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/07e93458
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/07e93458
Branch: refs/heads/master
Commit: 07e93458fe69b7701ea2045926b4afe30d166cb5
Parents: c18e7a9
Author: anastas
Authored: Thu Feb 15 14:34:12 2018 +0200
Committer: anastas
Committed: Thu Feb 15 14:34:12 2018 +0200
--
.../regionserver/ImmutableMemStoreLAB.java | 13 +++-
.../TestCompactingToCellFlatMapMemStore.java| 66
2 files changed, 78 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/07e93458/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
index 93d2685..871f526 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
@@ -45,9 +45,20 @@ public class ImmutableMemStoreLAB implements MemStoreLAB {
throw new IllegalStateException("This is an Immutable MemStoreLAB.");
}
+ /**
+ * The process of merging assumes all cells are allocated on mslab.
+ * There is a rare case in which the first immutable segment,
+ * participating in a merge, is a CSLM.
+ * Since the CSLM hasn't been flattened yet, and there is no point in
flattening it (since it is
+ * going to be merged), its big cells (for whom size > maxAlloc) must be
copied into mslab.
+ * This method copies the passed cell into the first mslab in the mslabs
list,
+ * returning either a new cell instance over the copied data,
+ * or null when this cell cannt be copied.
+ */
@Override
public Cell forceCopyOfBigCellInto(Cell cell) {
-throw new IllegalStateException("This is an Immutable MemStoreLAB.");
+MemStoreLAB mslab = this.mslabs.get(0);
+return mslab.forceCopyOfBigCellInto(cell);
}
/* Creating chunk to be used as index chunk in CellChunkMap, part of the
chunks array.
http://git-wip-us.apache.org/repos/asf/hbase/blob/07e93458/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 9b81c7f..a8561de 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellComparatorImpl;
@@ -823,6 +824,71 @@ public class TestCompactingToCellFlatMapMemStore extends
TestCompactingMemStore
assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize());
}
+ /**
+ * CellChunkMap Segment index requires all cell data to be written in the
MSLAB Chunks.
+ * Even though MSLAB is enabled, cells bigger than the size of a chunk are
not
+ * written in the MSLAB Chunks.
+ * If such cells are found in the process of a merge they need to be copied
into MSLAB.
+ * testForceCopyOfBigCellIntoImmutableSegment checks that the
+ * ImmutableMemStoreLAB's forceCopyOfBigCellInto does what it's supposed to
do.
+ */
+ @Test
+ public void testForceCopyOfBigCellIntoImmutableSegment() throws IOException {
+
+if (toCellChunkMap == false) {
+ return;
+}
+
+// set memstore to flat into CellChunkMap
+MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC;
+
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
+String.valueOf(compactionType));
+((MyCompactingMemStore) memstore).initiateType(compactionType,
memstore.getConfiguration());
+((CompactingMemStore)
memstore).setIndexType(C
