hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai)
Repository: hbase Updated Branches: refs/heads/branch-1 e30329e37 -> e8f056935 HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8f05693 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8f05693 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8f05693 Branch: refs/heads/branch-1 Commit: e8f056935ba90bb46b960d3a6086a99730c95abd Parents: e30329e Author: tedyu Authored: Sun Nov 20 18:17:21 2016 -0800 Committer: tedyu Committed: Sun Nov 20 18:17:21 2016 -0800 -- .../hadoop/hbase/regionserver/HRegion.java | 28 ++-- .../hbase/client/TestFromClientSide3.java | 139 ++- 2 files changed, 152 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e8f05693/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 4972fd3..8f8add8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7188,28 +7188,28 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } MultiVersionConcurrencyControl.WriteEntry writeEntry = null; -boolean locked; +boolean locked = false; boolean walSyncSuccessful = false; -List acquiredRowLocks; +List acquiredRowLocks = null; long addedSize = 0; List mutations = new ArrayList(); Collection rowsToLock = processor.getRowsToLock(); long mvccNum = 0; WALKey walKey = null; try { - // 2. Acquire the row lock(s) - acquiredRowLocks = new ArrayList(rowsToLock.size()); - for (byte[] row : rowsToLock) { -// Attempt to lock all involved rows, throw if any lock times out -// use a writer lock for mixed reads and writes -acquiredRowLocks.add(getRowLockInternal(row, false)); - } - // 3. Region lock - lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : acquiredRowLocks.size()); - locked = true; - - long now = EnvironmentEdgeManager.currentTime(); try { +// 2. Acquire the row lock(s) +acquiredRowLocks = new ArrayList(rowsToLock.size()); +for (byte[] row : rowsToLock) { + // Attempt to lock all involved rows, throw if any lock times out + // use a writer lock for mixed reads and writes + acquiredRowLocks.add(getRowLockInternal(row, false)); +} +// 3. Region lock +lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : acquiredRowLocks.size()); +locked = true; + +long now = EnvironmentEdgeManager.currentTime(); // 4. Let the processor scan the rows, generate mutations and add //waledits doProcessRowWithTimeout( http://git-wip-us.apache.org/repos/asf/hbase/blob/e8f05693/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index 09c7e86..08ccc42 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hbase.client; +import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -28,16 +29,36 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; - +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import static junit.framework.Assert.assertFalse; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org
hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai)
Repository: hbase Updated Branches: refs/heads/master ec9c9e201 -> bb645bcfd HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb645bcf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb645bcf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb645bcf Branch: refs/heads/master Commit: bb645bcfda74ad1f96b16e6e47543d44fbca5a98 Parents: ec9c9e2 Author: tedyu Authored: Sun Nov 20 07:09:02 2016 -0800 Committer: tedyu Committed: Sun Nov 20 07:09:02 2016 -0800 -- .../hadoop/hbase/regionserver/HRegion.java | 26 ++-- .../hbase/client/TestFromClientSide3.java | 124 +-- 2 files changed, 129 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bb645bcf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 4d35b51..c372faa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7044,8 +7044,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return; } -boolean locked; -List acquiredRowLocks; +boolean locked = false; +List acquiredRowLocks = null; List mutations = new ArrayList(); Collection rowsToLock = processor.getRowsToLock(); // This is assigned by mvcc either explicity in the below or in the guts of the WAL append @@ -7053,19 +7053,19 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi WriteEntry writeEntry = null; MemstoreSize memstoreSize = new MemstoreSize(); try { - // STEP 2. Acquire the row lock(s) - acquiredRowLocks = new ArrayList(rowsToLock.size()); - for (byte[] row : rowsToLock) { -// Attempt to lock all involved rows, throw if any lock times out -// use a writer lock for mixed reads and writes -acquiredRowLocks.add(getRowLockInternal(row, false)); - } - // STEP 3. Region lock - lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : acquiredRowLocks.size()); - locked = true; boolean success = false; - long now = EnvironmentEdgeManager.currentTime(); try { +// STEP 2. Acquire the row lock(s) +acquiredRowLocks = new ArrayList<>(rowsToLock.size()); +for (byte[] row : rowsToLock) { + // Attempt to lock all involved rows, throw if any lock times out + // use a writer lock for mixed reads and writes + acquiredRowLocks.add(getRowLockInternal(row, false)); +} +// STEP 3. Region lock +lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : acquiredRowLocks.size()); +locked = true; +long now = EnvironmentEdgeManager.currentTime(); // STEP 4. Let the processor scan the rows, generate mutations and add waledits doProcessRowWithTimeout(processor, now, this, mutations, walEdit, timeout); if (!mutations.isEmpty()) { http://git-wip-us.apache.org/repos/asf/hbase/blob/bb645bcf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index 82fbe77..cbc97a2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -23,13 +23,17 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; @@ -43,10 +47,17 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionScanne