[18/40] hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-26 Thread syuanjiang 
HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/49cba2c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/49cba2c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/49cba2c2

Branch: refs/heads/hbase-12439
Commit: 49cba2c237ecc1b3285d942f1ad176ea50c44cd1
Parents: 40cc666
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 11:22:23 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:50:09 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 104 +++
 2 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/49cba2c2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b21a84d..4836dc8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3392,6 +3392,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 applyFamilyMapToMemstore(familyMaps[i], memstoreSize);
   }
 
+  // update memstore size
+  this.addAndGetMemstoreSize(memstoreSize);
+
   // calling the post CP hook for batch mutation
   if (!replay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3444,7 +3447,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 } finally {
   // Call complete rather than completeAndWait because we probably had 
error if walKey != null
   if (writeEntry != null) mvcc.complete(writeEntry);
-  this.addAndGetMemstoreSize(memstoreSize);
   if (locked) {
 this.updatesLock.readLock().unlock();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/49cba2c2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..ae2f055
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.MemstoreSize;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the coprocessow is 
doing its work. To
+ * simulate this we call flush from the copr

hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-20 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 af9719e45 -> 93ac76ef6


HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93ac76ef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93ac76ef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93ac76ef

Branch: refs/heads/branch-1.1
Commit: 93ac76ef6164d8eb183f048ed727dc8b4290e0fa
Parents: af9719e
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 12:25:58 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:55:35 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 103 +++
 2 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/93ac76ef/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 7e5da03..3263873 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3207,6 +3207,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   }
 
   doRollBackMemstore = false;
+  // update memstore size
+  this.addAndGetGlobalMemstoreSize(addedSize);
+
   // calling the post CP hook for batch mutation
   if (!isInReplay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3252,7 +3255,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 rollbackMemstore(memstoreCells);
 if (writeEntry != null) mvcc.cancelMemstoreInsert(writeEntry);
   } else {
-this.addAndGetGlobalMemstoreSize(addedSize);
 if (writeEntry != null) {
   mvcc.completeMemstoreInsertWithSeqNum(writeEntry, walKey);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/93ac76ef/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..45600c7
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the coprocessow is 
doing its work. To
+ * simulate this we call flush from the cop

hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-20 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 d0d162d1f -> 0b5440d6d


HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b5440d6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b5440d6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b5440d6

Branch: refs/heads/branch-1.2
Commit: 0b5440d6d1a6fb1943917d68655b3abb8bd483b0
Parents: d0d162d
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 12:25:58 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:52:52 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 103 +++
 2 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0b5440d6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8f1a935..c6c611d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3219,6 +3219,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   }
 
   doRollBackMemstore = false;
+  // update memstore size
+  this.addAndGetGlobalMemstoreSize(addedSize);
+
   // calling the post CP hook for batch mutation
   if (!isInReplay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3276,7 +3279,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 if (writeEntry != null) mvcc.complete(writeEntry);
   } else {
-this.addAndGetGlobalMemstoreSize(addedSize);
 if (writeEntry != null) {
   mvcc.completeAndWait(writeEntry);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0b5440d6/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..45600c7
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the coprocessow is 
doing its work. To
+ * simulate this we call flush from the coprocessor itself
+ */
+@Category(LargeTests.class)
+public class Test

hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-20 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 acbba3749 -> 3dcbb733e


HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3dcbb733
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3dcbb733
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3dcbb733

Branch: refs/heads/branch-1.3
Commit: 3dcbb733e040f32e3f6bd9ab4063f5efe6bd522b
Parents: acbba37
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 12:25:58 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:51:53 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 103 +++
 2 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3dcbb733/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 8345e8d..15c906f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3345,6 +3345,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   }
 
   doRollBackMemstore = false;
+  // update memstore size
+  this.addAndGetGlobalMemstoreSize(addedSize);
+
   // calling the post CP hook for batch mutation
   if (!isInReplay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3402,7 +3405,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 if (writeEntry != null) mvcc.complete(writeEntry);
   } else {
-this.addAndGetGlobalMemstoreSize(addedSize);
 if (writeEntry != null) {
   mvcc.completeAndWait(writeEntry);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3dcbb733/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..45600c7
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the coprocessow is 
doing its work. To
+ * simulate this we call flush from the coprocessor itself
+ */
+@Category(LargeTests.class)
+public class Test

hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-20 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 6ea691524 -> d69a6366f


HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d69a6366
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d69a6366
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d69a6366

Branch: refs/heads/branch-1
Commit: d69a6366f6d36ce229df80447998e71ca4654518
Parents: 6ea6915
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 12:25:58 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:50:53 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 103 +++
 2 files changed, 106 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d69a6366/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 81547d5..d16af6d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3503,6 +3503,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   }
 
   doRollBackMemstore = false;
+  // update memstore size
+  this.addAndGetGlobalMemstoreSize(addedSize);
+
   // calling the post CP hook for batch mutation
   if (!isInReplay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3560,7 +3563,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 if (writeEntry != null) mvcc.complete(writeEntry);
   } else {
-this.addAndGetGlobalMemstoreSize(addedSize);
 if (writeEntry != null) {
   mvcc.completeAndWait(writeEntry);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d69a6366/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..45600c7
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the coprocessow is 
doing its work. To
+ * simulate this we call flush from the coprocessor itself
+ */
+@Category(LargeTests.class)
+public class TestNega

hbase git commit: HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete Coprocessor call

2017-04-20 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/master 40cc666ac -> 49cba2c23


HBASE-17937 Memstore size becomes negative in case of expensive postPut/Delete 
Coprocessor call

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/49cba2c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/49cba2c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/49cba2c2

Branch: refs/heads/master
Commit: 49cba2c237ecc1b3285d942f1ad176ea50c44cd1
Parents: 40cc666
Author: Abhishek Singh Chouhan 
Authored: Wed Apr 19 11:22:23 2017 +0530
Committer: zhangduo 
Committed: Fri Apr 21 08:50:09 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |   4 +-
 ...NegativeMemstoreSizeWithSlowCoprocessor.java | 104 +++
 2 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/49cba2c2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b21a84d..4836dc8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3392,6 +3392,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 applyFamilyMapToMemstore(familyMaps[i], memstoreSize);
   }
 
+  // update memstore size
+  this.addAndGetMemstoreSize(memstoreSize);
+
   // calling the post CP hook for batch mutation
   if (!replay && coprocessorHost != null) {
 MiniBatchOperationInProgress miniBatchOp =
@@ -3444,7 +3447,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 } finally {
   // Call complete rather than completeAndWait because we probably had 
error if walKey != null
   if (writeEntry != null) mvcc.complete(writeEntry);
-  this.addAndGetMemstoreSize(memstoreSize);
   if (locked) {
 this.updatesLock.readLock().unlock();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/49cba2c2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
new file mode 100644
index 000..ae2f055
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestNegativeMemstoreSizeWithSlowCoprocessor.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache 
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the 
License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless 
required by applicable
+ * law or agreed to in writing, software distributed under the License is 
distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 
implied. See the License
+ * for the specific language governing permissions and limitations under the 
License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.MemstoreSize;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test that verifies we do not have memstore size negative when a 
postPut/Delete hook is
+ * slow/expensive and a flush is triggered at the same time the copr