[33/34] hbase git commit: HBASE-19865 Add UT for sync replication peer in DA state

[zhangduo]

HBASE-19865 Add UT for sync replication peer in DA state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0567fbb4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0567fbb4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0567fbb4

Branch: refs/heads/HBASE-19064
Commit: 0567fbb4b6117c10c8d6f0563b33d363eaf39a1a
Parents: 05e8443
Author: zhangduo 
Authored: Tue May 8 20:33:22 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/replication/TestReplicationBase.java  | 28 +++---
 ...estReplicationChangingPeerRegionservers.java | 20 ++
 .../TestReplicationSmallTestsSync.java  | 40 
 3 files changed, 76 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0567fbb4/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index b2c5aef..4a4b358 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,6 +27,8 @@ import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -58,6 +59,9 @@ import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
 /**
  * This class is only a base for other integration-level replication tests.
  * Do not add tests here.
@@ -99,6 +103,10 @@ public class TestReplicationBase {
 return false;
   }
 
+  protected boolean isSyncPeer() {
+return false;
+  }
+
   protected final void cleanUp() throws IOException, InterruptedException {
 // Starting and stopping replication can make us miss new logs,
 // rolling like this makes sure the most recent one gets added to the queue
@@ -245,9 +253,19 @@ public class TestReplicationBase {
   @Before
   public void setUpBase() throws IOException {
 if (!peerExist(PEER_ID2)) {
-  ReplicationPeerConfig rpc = ReplicationPeerConfig.newBuilder()
-  
.setClusterKey(utility2.getClusterKey()).setSerial(isSerialPeer()).build();
-  hbaseAdmin.addReplicationPeer(PEER_ID2, rpc);
+  ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder()
+.setClusterKey(utility2.getClusterKey()).setSerial(isSerialPeer());
+  if (isSyncPeer()) {
+FileSystem fs2 = utility2.getTestFileSystem();
+// The remote wal dir is not important as we do not use it in DA 
state, here we only need to
+// confirm that a sync peer in DA state can still replicate data to 
remote cluster
+// asynchronously.
+builder.setReplicateAllUserTables(false)
+  .setTableCFsMap(ImmutableMap.of(tableName, ImmutableList.of()))
+  .setRemoteWALDir(new Path("/RemoteWAL")
+.makeQualified(fs2.getUri(), 
fs2.getWorkingDirectory()).toUri().toString());
+  }
+  hbaseAdmin.addReplicationPeer(PEER_ID2, builder.build());
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/0567fbb4/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java
index b94b443..5c96742 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java
@@ -62,22 +62,28 @@ public class TestReplicationChangingPeerRegionservers 
extends TestReplicationBas
   private static final Logger LOG =
   LoggerFactory.getLogger(TestRepli

[05/34] hbase git commit: HBASE-20544 Make HBTU default to random ports.

[zhangduo]

HBASE-20544 Make HBTU default to random ports.

Signed-off-by: Umesh Agashe 
Signed-off-by: Josh Elser 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8ba2a7ee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8ba2a7ee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8ba2a7ee

Branch: refs/heads/HBASE-19064
Commit: 8ba2a7eeb967a80d04020e30b2fe8e8652ea1758
Parents: a67909d
Author: Sean Busbey 
Authored: Wed May 9 23:23:29 2018 -0700
Committer: Sean Busbey 
Committed: Wed May 9 23:35:20 2018 -0700

--
 .../src/test/resources/hbase-site.xml   |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 hbase-backup/src/test/resources/hbase-site.xml  |  7 --
 hbase-client/src/test/resources/hbase-site.xml  |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 hbase-rest/src/test/resources/hbase-site.xml|  7 --
 .../hadoop/hbase/rsgroup/TestEnableRSGroup.java |  4 +-
 hbase-rsgroup/src/test/resources/hbase-site.xml |  7 --
 .../apache/hadoop/hbase/LocalHBaseCluster.java  | 27 ++--
 .../hadoop/hbase/HBaseTestingUtility.java   | 18 ++
 .../apache/hadoop/hbase/MiniHBaseCluster.java   |  6 --
 .../hadoop/hbase/TestHBaseTestingUtility.java   | 33 +++---
 hbase-server/src/test/resources/hbase-site.xml  |  7 --
 hbase-shell/src/test/resources/hbase-site.xml   |  7 --
 .../src/test/resources/hbase-site.xml   |  7 --
 hbase-spark/src/test/resources/hbase-site.xml   |  7 --
 .../hbase/TestHBaseTestingUtilSpinup.java   | 67 
 hbase-thrift/src/test/resources/hbase-site.xml  |  7 --
 21 files changed, 117 insertions(+), 143 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8ba2a7ee/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml
--
diff --git 
a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml 
b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml
index 858d428..99d2ab8 100644
--- a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml
+++ b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml
@@ -29,11 +29,4 @@
 hbase.hconnection.threads.keepalivetime
 3
   
-  
-hbase.localcluster.assign.random.ports
-true
-
-  Assign random ports to master and RS info server (UI).
-
-  
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8ba2a7ee/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
--
diff --git 
a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
 
b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
index 858d428..99d2ab8 100644
--- 
a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
+++ 
b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
@@ -29,11 +29,4 @@
 hbase.hconnection.threads.keepalivetime
 3
   
-  
-hbase.localcluster.assign.random.ports
-true
-
-  Assign random ports to master and RS info server (UI).
-
-  
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8ba2a7ee/hbase-backup/src/test/resources/hbase-site.xml
--
diff --git a/hbase-backup/src/test/resources/hbase-site.xml 
b/hbase-backup/src/test/resources/hbase-site.xml
index 858d428..99d2ab8 100644
--- a/hbase-backup/src/test/resources/hbase-site.xml
+++ b/hbase-backup/src/test/resources/hbase-site.xml
@@ -29,11 +29,4 @@
 hbase.hconnection.threads.keepalivetime
 3
   
-  
-hbase.localcluster.assign.random.ports
-true
-
-  Assign random ports to master and RS info server (UI).
-
-  
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8ba2a7ee/hbase-client/src/test/resources/hbase-site.xml
--
diff --git a/hbase-client/src/test/resources/hbase-site.xml 
b/hbase-client/src/test/resources/hbase-site.xml
index 858d428..99d2ab8 100644
--- a/hbase-client/src/test/resources/hbase-site.xml
+++ b/hbase-client/src/test/resources/hbase-site.xml
@@ -29,11 +29,4 @@
 hbase.hconnection.threads.keepalivetime
 3
   
-  
-hbase.localcluster.assign.random.ports
-true
-
-  Assign random ports to master and RS info server (UI).
-
-  
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8ba2a7ee/hbase-endpoint/src/test/resources/hbase-site.xml

[08/34] hbase git commit: HBASE-20411 Ameliorate MutableSegment synchronize

[zhangduo]

HBASE-20411 Ameliorate MutableSegment synchronize

Change the MemStore size accounting so we don't synchronize across three
volatiles applying deltas. Instead:

 + Make MemStoreSize, a datastructure of our memstore size longs, immutable.
 + Undo MemStoreSizing being an instance of MemStoreSize; instead it has-a.
 + Make two MemStoreSizing implementations; one thread-safe, the other not.
 + Let all memory sizing longs run independent, untied by
   synchronize (Huaxiang and Anoop suggestion) using atomiclongs.
 + Review all use of MemStoreSizing. Many are single-threaded and do
   not need to be synchronized; use the non-thread safe counter.

TODO: Use this technique accounting at the global level too.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/021f66d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/021f66d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/021f66d1

Branch: refs/heads/HBASE-19064
Commit: 021f66d11d2cbb7308308093e29e69d6e7661ee9
Parents: c60578d
Author: Michael Stack 
Authored: Sun May 6 21:29:49 2018 -0700
Committer: Michael Stack 
Committed: Sat May 12 02:17:50 2018 +0100

--
 .../hbase/regionserver/AbstractMemStore.java|   8 +-
 .../regionserver/CSLMImmutableSegment.java  |   2 +-
 .../regionserver/CellArrayImmutableSegment.java |   6 +-
 .../regionserver/CellChunkImmutableSegment.java |  14 +-
 .../hbase/regionserver/CompactingMemStore.java  |  38 ++---
 .../hbase/regionserver/CompactionPipeline.java  |  44 ++---
 .../regionserver/CompositeImmutableSegment.java |  10 +-
 .../hbase/regionserver/DefaultMemStore.java |  39 ++---
 .../hadoop/hbase/regionserver/HRegion.java  | 162 ++-
 .../hadoop/hbase/regionserver/HStore.java   |   2 +-
 .../hadoop/hbase/regionserver/MemStore.java |   3 +-
 .../hbase/regionserver/MemStoreFlusher.java |  21 ++-
 .../hadoop/hbase/regionserver/MemStoreSize.java |  76 -
 .../hbase/regionserver/MemStoreSizing.java  | 113 -
 .../hbase/regionserver/MutableSegment.java  |   4 +-
 .../NonThreadSafeMemStoreSizing.java|  81 ++
 .../regionserver/RegionServerAccounting.java|  33 ++--
 .../regionserver/RegionServicesForStores.java   |   4 +-
 .../hbase/regionserver/ScannerContext.java  |   3 -
 .../hadoop/hbase/regionserver/Segment.java  |  64 
 .../regionserver/ThreadSafeMemStoreSizing.java  |  80 +
 .../regionserver/TestCompactingMemStore.java|  44 ++---
 .../TestCompactingToCellFlatMapMemStore.java|  43 ++---
 .../hbase/regionserver/TestDefaultMemStore.java |  13 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |  12 +-
 .../regionserver/TestHRegionReplayEvents.java   |  28 ++--
 .../TestHRegionWithInMemoryFlush.java   |   2 +-
 .../hadoop/hbase/regionserver/TestHStore.java   |  41 +++--
 28 files changed, 595 insertions(+), 395 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/021f66d1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 9f4fd2f..b82afba 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -165,13 +165,7 @@ public abstract class AbstractMemStore implements MemStore 
{
 
   @Override
   public MemStoreSize getSnapshotSize() {
-return getSnapshotSizing();
-  }
-
-  MemStoreSizing getSnapshotSizing() {
-return new MemStoreSizing(this.snapshot.keySize(),
-this.snapshot.heapSize(),
-this.snapshot.offHeapSize());
+return this.snapshot.getMemStoreSize();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/021f66d1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
index 6af84cb..9e206ea 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
@@ -40,7 +40,7 @@ public class CSLMImmutableSegment extends ImmutableSegment {
 super(segment);
 // update the segment metadata heap size
 long indexOverhead = -MutableSegment.DEEP_OVERHEAD + DEEP_OVERHEAD

[02/34] hbase git commit: HBASE-20204 Add locking to RefreshFileConnections in BucketCache

[zhangduo]

HBASE-20204 Add locking to RefreshFileConnections in BucketCache

This is a follow-up to HBASE-20141 where Anoop suggested adding locking
for refreshing channels.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9c046c09
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9c046c09
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9c046c09

Branch: refs/heads/HBASE-19064
Commit: 9c046c091c9b1ca6eb50d8b3e945c24a841dd4f1
Parents: 4f2dfd3
Author: Zach York 
Authored: Wed Mar 14 15:38:22 2018 -0700
Committer: Zach York 
Committed: Wed May 9 14:01:51 2018 -0700

--
 .../hbase/io/hfile/bucket/FileIOEngine.java | 32 +++-
 .../hbase/io/hfile/bucket/TestFileIOEngine.java | 19 +---
 2 files changed, 38 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9c046c09/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index 29b810f..0710d26 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -26,6 +26,7 @@ import java.nio.channels.ClosedByInterruptException;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
 import java.util.Arrays;
+import java.util.concurrent.locks.ReentrantLock;
 import org.apache.hadoop.hbase.io.hfile.Cacheable;
 import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
 import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
@@ -49,6 +50,7 @@ public class FileIOEngine implements IOEngine {
   private final String[] filePaths;
   private final FileChannel[] fileChannels;
   private final RandomAccessFile[] rafs;
+  private final ReentrantLock[] channelLocks;
 
   private final long sizePerFile;
   private final long capacity;
@@ -75,6 +77,7 @@ public class FileIOEngine implements IOEngine {
   }
 }
 this.rafs = new RandomAccessFile[filePaths.length];
+this.channelLocks = new ReentrantLock[filePaths.length];
 for (int i = 0; i < filePaths.length; i++) {
   String filePath = filePaths[i];
   try {
@@ -90,6 +93,7 @@ public class FileIOEngine implements IOEngine {
 }
 rafs[i].setLength(sizePerFile);
 fileChannels[i] = rafs[i].getChannel();
+channelLocks[i] = new ReentrantLock();
 LOG.info("Allocating cache " + StringUtils.byteDesc(sizePerFile)
 + ", on the path:" + filePath);
   } catch (IOException fex) {
@@ -233,8 +237,7 @@ public class FileIOEngine implements IOEngine {
   } catch (ClosedByInterruptException e) {
 throw e;
   } catch (ClosedChannelException e) {
-LOG.warn("Caught ClosedChannelException accessing BucketCache, 
reopening file. ", e);
-refreshFileConnection(accessFileNum);
+refreshFileConnection(accessFileNum, e);
 continue;
   }
   // recover the limit
@@ -282,13 +285,26 @@ public class FileIOEngine implements IOEngine {
   }
 
   @VisibleForTesting
-  void refreshFileConnection(int accessFileNum) throws IOException {
-FileChannel fileChannel = fileChannels[accessFileNum];
-if (fileChannel != null) {
-  fileChannel.close();
+  void refreshFileConnection(int accessFileNum, IOException ioe) throws 
IOException {
+ReentrantLock channelLock = channelLocks[accessFileNum];
+channelLock.lock();
+try {
+  FileChannel fileChannel = fileChannels[accessFileNum];
+  if (fileChannel != null) {
+// Don't re-open a channel if we were waiting on another
+// thread to re-open the channel and it is now open.
+if (fileChannel.isOpen()) {
+  return;
+}
+fileChannel.close();
+  }
+  LOG.warn("Caught ClosedChannelException accessing BucketCache, reopening 
file: "
+  + filePaths[accessFileNum], ioe);
+  rafs[accessFileNum] = new RandomAccessFile(filePaths[accessFileNum], 
"rw");
+  fileChannels[accessFileNum] = rafs[accessFileNum].getChannel();
+} finally{
+  channelLock.unlock();
 }
-rafs[accessFileNum] = new RandomAccessFile(filePaths[accessFileNum], "rw");
-fileChannels[accessFileNum] = rafs[accessFileNum].getChannel();
   }
 
   private static interface FileAccessor {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9c046c09/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git 
a

[07/34] hbase git commit: HBASE-20411 Ameliorate MutableSegment synchronize

[zhangduo]

http://git-wip-us.apache.org/repos/asf/hbase/blob/021f66d1/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index 5cbfff9..8dbddb9 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -355,7 +355,7 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
   private long runSnapshot(final AbstractMemStore hmc, boolean useForce)
   throws IOException {
 // Save off old state.
-long oldHistorySize = hmc.getSnapshot().keySize();
+long oldHistorySize = hmc.getSnapshot().getDataSize();
 long prevTimeStamp = hmc.timeOfOldestEdit();
 
 hmc.snapshot();
@@ -616,9 +616,10 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize());
 assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
 
-MemStoreSize size = memstore.getFlushableSize();
+MemStoreSize mss = memstore.getFlushableSize();
 MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
-region.decrMemStoreSize(size);  // simulate flusher
+// simulate flusher
+region.decrMemStoreSize(mss);
 ImmutableSegment s = memstore.getSnapshot();
 assertEquals(4, s.getCellsCount());
 assertEquals(0, regionServicesForStores.getMemStoreSize());
@@ -667,7 +668,7 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 assertEquals(totalCellsLen1 + totalCellsLen2, 
regionServicesForStores.getMemStoreSize());
 assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize());
 
-MemStoreSize size = memstore.getFlushableSize();
+MemStoreSize mss = memstore.getFlushableSize();
 ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline 
and compact
 assertEquals(0, memstore.getSnapshot().getCellsCount());
 assertEquals(totalCellsLen1 + totalCellsLen2, 
regionServicesForStores.getMemStoreSize());
@@ -675,9 +676,10 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 + 7 * oneCellOnCAHeapSize;
 assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
 
-size = memstore.getFlushableSize();
+mss = memstore.getFlushableSize();
 MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
-region.decrMemStoreSize(size);  // simulate flusher
+// simulate flusher
+region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), 
mss.getOffHeapSize());
 ImmutableSegment s = memstore.getSnapshot();
 assertEquals(7, s.getCellsCount());
 assertEquals(0, regionServicesForStores.getMemStoreSize());
@@ -722,7 +724,7 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 assertEquals(totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
 
 ((MyCompactingMemStore) memstore).disableCompaction();
-MemStoreSize size = memstore.getFlushableSize();
+MemStoreSize mss = memstore.getFlushableSize();
 ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline 
without compaction
 assertEquals(0, memstore.getSnapshot().getCellsCount());
 // No change in the cells data size. ie. memstore size. as there is no 
compaction.
@@ -738,7 +740,7 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 assertEquals(totalHeapSize3, ((CompactingMemStore) memstore).heapSize());
 
 ((MyCompactingMemStore)memstore).enableCompaction();
-size = memstore.getFlushableSize();
+mss = memstore.getFlushableSize();
 ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline 
and compact
 assertEquals(0, memstore.getSnapshot().getCellsCount());
 // active flushed to pipeline and all 3 segments compacted. Will get rid 
of duplicated cells.
@@ -751,9 +753,10 @@ public class TestCompactingMemStore extends 
TestDefaultMemStore {
 assertEquals(4 * oneCellOnCAHeapSize + MutableSegment.DEEP_OVERHEAD
 + CellArrayImmutableSegment.DEEP_OVERHEAD_CAM, ((CompactingMemStore) 
memstore).heapSize());
 
-size = memstore.getFlushableSize();
+mss = memstore.getFlushableSize();
 MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
-region.decrMemStoreSize(size);  // simulate flusher
+// simulate flusher
+region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), 
mss.getOffHeapSize());
 ImmutableSegment s = memstore.getSnapshot();
 assertEquals(4, s.getCellsCount());
 assertEquals(0, regionServicesForStores.getMemStoreSize());
@@ -811,9 +814,10 @@ public c

[27/34] hbase git commit: HBASE-20434 Also remove remote wals when peer is in DA state

[zhangduo]

HBASE-20434 Also remove remote wals when peer is in DA state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ccef1fad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ccef1fad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ccef1fad

Branch: refs/heads/HBASE-19064
Commit: ccef1fad6956820dd2b5685ba4f7691f9f61764e
Parents: 3970947
Author: zhangduo 
Authored: Wed Apr 25 17:12:23 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/replication/ReplicationUtils.java |   4 +
 ...ransitPeerSyncReplicationStateProcedure.java |   2 +-
 .../regionserver/ReplicationSource.java |   7 +-
 .../regionserver/ReplicationSourceManager.java  |  86 ++--
 .../hadoop/hbase/wal/AbstractFSWALProvider.java |  19 ++--
 .../hbase/wal/SyncReplicationWALProvider.java   |  30 +-
 .../TestSyncReplicationRemoveRemoteWAL.java | 101 +++
 .../TestReplicationSourceManager.java   |  68 -
 8 files changed, 251 insertions(+), 66 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ccef1fad/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 66e9b01..069db7a 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -191,6 +191,10 @@ public final class ReplicationUtils {
 return new Path(remoteWALDir, peerId);
   }
 
+  public static Path getRemoteWALDirForPeer(Path remoteWALDir, String peerId) {
+return new Path(remoteWALDir, peerId);
+  }
+
   /**
* Do the sleeping logic
* @param msg Why we sleep

http://git-wip-us.apache.org/repos/asf/hbase/blob/ccef1fad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
index 5da2b0c..99fd615 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
@@ -211,7 +211,7 @@ public class TransitPeerSyncReplicationStateProcedure
   case CREATE_DIR_FOR_REMOTE_WAL:
 MasterFileSystem mfs = env.getMasterFileSystem();
 Path remoteWALDir = new Path(mfs.getWALRootDir(), 
ReplicationUtils.REMOTE_WAL_DIR_NAME);
-Path remoteWALDirForPeer = new Path(remoteWALDir, peerId);
+Path remoteWALDirForPeer = 
ReplicationUtils.getRemoteWALDirForPeer(remoteWALDir, peerId);
 FileSystem walFs = mfs.getWALFileSystem();
 try {
   if (walFs.exists(remoteWALDirForPeer)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ccef1fad/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 1a27fc1..7313f13 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -549,14 +549,17 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 }
 
 /**
+ * 
  * Split a path to get the start time
+ * 
+ * 
  * For example: 10.20.20.171%3A60020.1277499063250
+ * 
  * @param p path to split
  * @return start time
  */
 private static long getTS(Path p) {
-  int tsIndex = p.getName().lastIndexOf('.') + 1;
-  return Long.parseLong(p.getName().substring(tsIndex));
+  return AbstractFSWALProvider.getWALStartTimeFromWALName(p.getName());
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/ccef1fad/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java

[11/34] hbase git commit: HBASE-19990 Create remote wal directory when transitting to state S

[zhangduo]

HBASE-19990 Create remote wal directory when transitting to state S


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/280a80e6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/280a80e6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/280a80e6

Branch: refs/heads/HBASE-19064
Commit: 280a80e6933ee7cd87aa1dcb1abd8c825e80cf76
Parents: 2c7a90a
Author: zhangduo 
Authored: Wed Feb 14 16:01:16 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../procedure2/ProcedureYieldException.java |  9 --
 .../hbase/replication/ReplicationUtils.java |  2 ++
 .../hadoop/hbase/master/MasterFileSystem.java   | 19 ++---
 .../master/procedure/MasterProcedureEnv.java|  5 
 ...ransitPeerSyncReplicationStateProcedure.java | 29 
 .../hbase/replication/TestSyncReplication.java  |  8 ++
 6 files changed, 55 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/280a80e6/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
--
diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
index 0487ac5b..dbb9981 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
@@ -15,16 +15,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hadoop.hbase.procedure2;
 
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 
-// TODO: Not used yet
+/**
+ * Indicate that a procedure wants to be rescheduled. Usually because there 
are something wrong but
+ * we do not want to fail the procedure.
+ * 
+ * TODO: need to support scheduling after a delay.
+ */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
 public class ProcedureYieldException extends ProcedureException {
+
   /** default constructor */
   public ProcedureYieldException() {
 super();

http://git-wip-us.apache.org/repos/asf/hbase/blob/280a80e6/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index d94cb00..e402d0f 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -41,6 +41,8 @@ public final class ReplicationUtils {
 
   public static final String REPLICATION_ATTR_NAME = "__rep__";
 
+  public static final String REMOTE_WAL_DIR_NAME = "remoteWALs";
+
   private ReplicationUtils() {
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/280a80e6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 864be02..7ccbd71 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.log.HBaseMarkers;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -133,7 +134,6 @@ public class MasterFileSystem {
* Idempotent.
*/
   private void createInitialFileSystemLayout() throws IOException {
-
 final String[] protectedSubDirs = new String[] {
 HConstants.BASE_NAMESPACE_DIR,
 HConstants.HFILE_ARCHIVE_DIRECTORY,
@@ -145,7 +145,8 @@ public class MasterFileSystem {
   HConstants.HREGION_LOGDIR_NAME,
   HConstants.HREGION_OLDLOGDIR_NAME,
   HConstants.CORRUPT_DIR_NAME,
-  WALProcedureStore.MASTER_PROCEDURE_LOGDIR
+  WALProcedureStore.MASTER_PROCEDURE_LOGDIR,
+   

[16/34] hbase git commit: HBASE-19781 Add a new cluster state flag for synchronous replication

[zhangduo]

http://git-wip-us.apache.org/repos/asf/hbase/blob/11a1f32c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
index 8911982..f5eca39 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
@@ -28,6 +28,7 @@ import 
org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
 import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
 import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.ReplicationTests;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
@@ -67,9 +68,9 @@ public class TestHBaseFsckReplication {
 String peerId1 = "1";
 String peerId2 = "2";
 peerStorage.addPeer(peerId1, 
ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
-  true);
+  true, SyncReplicationState.NONE);
 peerStorage.addPeer(peerId2, 
ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
-  true);
+  true, SyncReplicationState.NONE);
 for (int i = 0; i < 10; i++) {
   queueStorage.addWAL(ServerName.valueOf("localhost", 1 + i, 10 + 
i), peerId1,
 "file-" + i);

http://git-wip-us.apache.org/repos/asf/hbase/blob/11a1f32c/hbase-shell/src/main/ruby/hbase/replication_admin.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/replication_admin.rb 
b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
index d1f1344..5f86365 100644
--- a/hbase-shell/src/main/ruby/hbase/replication_admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
@@ -20,6 +20,7 @@
 include Java
 
 java_import 
org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil
+java_import org.apache.hadoop.hbase.replication.SyncReplicationState
 java_import org.apache.hadoop.hbase.replication.ReplicationPeerConfig
 java_import org.apache.hadoop.hbase.util.Bytes
 java_import org.apache.hadoop.hbase.zookeeper.ZKConfig
@@ -338,6 +339,20 @@ module Hbase
   '!' + ReplicationPeerConfigUtil.convertToString(tableCFs)
 end
 
+# Transit current cluster to a new state in the specified synchronous
+# replication peer
+def transit_peer_sync_replication_state(id, state)
+  if 'ACTIVE'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id, 
SyncReplicationState::ACTIVE)
+  elsif 'DOWNGRADE_ACTIVE'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id, 
SyncReplicationState::DOWNGRADE_ACTIVE)
+  elsif 'STANDBY'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id, 
SyncReplicationState::STANDBY)
+  else
+raise(ArgumentError, 'synchronous replication state must be ACTIVE, 
DOWNGRADE_ACTIVE or STANDBY')
+  end
+end
+
 
#--
 # Enables a table's replication switch
 def enable_tablerep(table_name)

http://git-wip-us.apache.org/repos/asf/hbase/blob/11a1f32c/hbase-shell/src/main/ruby/shell.rb
--
diff --git a/hbase-shell/src/main/ruby/shell.rb 
b/hbase-shell/src/main/ruby/shell.rb
index ab07a79..ba4d154 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -391,6 +391,7 @@ Shell.load_command_group(
 get_peer_config
 list_peer_configs
 update_peer_config
+transit_peer_sync_replication_state
   ]
 )
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/11a1f32c/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb 
b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
index f3ab749..f2ec014 100644
--- a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
@@ -39,8 +39,8 @@ EOF
 peers = replication_admin.list_peers
 
 formatter.header(%w[PEER_ID CLUSTER_KEY ENDPOINT_CLASSNAME
-REMOTE_ROOT_DIR STATE REPLICATE_ALL 
-NAMESPACES TABLE_CFS BANDWIDTH
+REMOTE_ROOT_DIR SYNC_REPLICATION_STATE STATE
+REPLICATE_ALL NAMESPACES TABLE_CFS BANDWIDTH

[24/34] hbase git commit: HBASE-19082 Reject read/write from client but accept write from replication in state S

[zhangduo]

HBASE-19082 Reject read/write from client but accept write from replication in 
state S


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2c7a90a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2c7a90a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2c7a90a7

Branch: refs/heads/HBASE-19064
Commit: 2c7a90a7b4b5c320806cf3174fd3938424f18164
Parents: 61384ec
Author: zhangduo 
Authored: Mon Feb 12 18:20:18 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   3 -
 .../src/main/protobuf/MasterProcedure.proto |   3 +-
 .../hbase/replication/ReplicationUtils.java |   4 +
 ...ransitPeerSyncReplicationStateProcedure.java |  10 +
 .../hadoop/hbase/regionserver/HRegion.java  |   5 +-
 .../hbase/regionserver/HRegionServer.java   |   2 +-
 .../hbase/regionserver/RSRpcServices.java   |  88 ++--
 .../RejectRequestsFromClientStateChecker.java   |  44 
 .../regionserver/ReplicationSink.java   |  72 ---
 .../SyncReplicationPeerInfoProvider.java|  10 +-
 .../SyncReplicationPeerInfoProviderImpl.java|  19 +-
 .../hbase/wal/SyncReplicationWALProvider.java   |   3 +
 .../org/apache/hadoop/hbase/wal/WALFactory.java |   4 +-
 .../hbase/replication/TestSyncReplication.java  | 200 +++
 .../wal/TestSyncReplicationWALProvider.java |   8 +-
 15 files changed, 401 insertions(+), 74 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2c7a90a7/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 522c2cf..9241682 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1355,9 +1355,6 @@ public final class HConstants {
 
   public static final String NOT_IMPLEMENTED = "Not implemented";
 
-  // TODO: need to find a better place to hold it.
-  public static final String SYNC_REPLICATION_ENABLED = 
"hbase.replication.sync.enabled";
-
   private HConstants() {
 // Can't be instantiated with this ctor.
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c7a90a7/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto 
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 67c1b43..e8b940e 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -397,7 +397,8 @@ enum PeerSyncReplicationStateTransitionState {
   REOPEN_ALL_REGIONS_IN_PEER = 5;
   TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE = 6;
   REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END = 7;
-  POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 8;
+  CREATE_DIR_FOR_REMOTE_WAL = 8;
+  POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 9;
 }
 
 message PeerModificationStateData {

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c7a90a7/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index e4dea83..d94cb00 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -37,6 +37,10 @@ import org.apache.yetus.audience.InterfaceAudience;
 @InterfaceAudience.Private
 public final class ReplicationUtils {
 
+  public static final String SYNC_REPLICATION_ENABLED = 
"hbase.replication.sync.enabled";
+
+  public static final String REPLICATION_ATTR_NAME = "__rep__";
+
   private ReplicationUtils() {
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c7a90a7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
index 8fc932f..69404a0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replica

[23/34] hbase git commit: HBASE-19943 Only allow removing sync replication peer which is in DA state

[zhangduo]

HBASE-19943 Only allow removing sync replication peer which is in DA state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8f59b26
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8f59b26
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8f59b26

Branch: refs/heads/HBASE-19064
Commit: f8f59b26cfa8b50c215d4d4d89b33e0218544620
Parents: 280a80e
Author: huzheng 
Authored: Thu Mar 1 18:34:02 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/ReplicationPeerManager.java | 14 -
 .../hbase/wal/SyncReplicationWALProvider.java   |  2 +-
 .../replication/TestReplicationAdmin.java   | 63 
 .../hbase/replication/TestSyncReplication.java  |  2 +-
 4 files changed, 78 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f8f59b26/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 0dc922d..41dd6e3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -120,8 +120,20 @@ public class ReplicationPeerManager {
 return desc;
   }
 
+  private void checkPeerInDAStateIfSyncReplication(String peerId) throws 
DoNotRetryIOException {
+ReplicationPeerDescription desc = peers.get(peerId);
+if (desc != null && desc.getPeerConfig().isSyncReplication()
+&& 
!SyncReplicationState.DOWNGRADE_ACTIVE.equals(desc.getSyncReplicationState())) {
+  throw new DoNotRetryIOException("Couldn't remove synchronous replication 
peer with state="
+  + desc.getSyncReplicationState()
+  + ", Transit the synchronous replication state to be 
DOWNGRADE_ACTIVE firstly.");
+}
+  }
+
   ReplicationPeerConfig preRemovePeer(String peerId) throws 
DoNotRetryIOException {
-return checkPeerExists(peerId).getPeerConfig();
+ReplicationPeerDescription pd = checkPeerExists(peerId);
+checkPeerInDAStateIfSyncReplication(peerId);
+return pd.getPeerConfig();
   }
 
   void preEnablePeer(String peerId) throws DoNotRetryIOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8f59b26/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
index ac4b4cd..282aa21 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
@@ -142,7 +142,7 @@ public class SyncReplicationWALProvider implements 
WALProvider, PeerActionListen
   @Override
   public WAL getWAL(RegionInfo region) throws IOException {
 if (region == null) {
-  return provider.getWAL(region);
+  return provider.getWAL(null);
 }
 Optional> peerIdAndRemoteWALDir =
   peerInfoProvider.getPeerIdAndRemoteWALDir(region);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8f59b26/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 0ad476f..486ab51 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -254,6 +254,62 @@ public class TestReplicationAdmin {
   }
 
   @Test
+  public void testRemovePeerWithNonDAState() throws Exception {
+TableName tableName = TableName.valueOf(name.getMethodName());
+TEST_UTIL.createTable(tableName, Bytes.toBytes("family"));
+ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
+
+String rootDir = "hdfs://srv1:/hbase";
+builder.setClusterKey(KEY_ONE);
+builder.setRemoteWALDir(rootDir);
+builder.setReplicateAllUserTables(false);
+Map> tableCfs = new HashMap<>();
+tableCfs.put(tableName, new ArrayList<>());
+builder.se

[20/34] hbase git commit: HBASE-19935 Only allow table replication for sync replication for now

[zhangduo]

HBASE-19935 Only allow table replication for sync replication for now


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/78c1d15a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/78c1d15a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/78c1d15a

Branch: refs/heads/HBASE-19064
Commit: 78c1d15a21d4bd003f737712878009f525343e24
Parents: 9a81f28
Author: Guanghao Zhang 
Authored: Tue Feb 6 16:00:59 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/ReplicationPeerConfig.java  |  9 +++
 .../replication/ReplicationPeerManager.java | 34 -
 .../replication/TestReplicationAdmin.java   | 73 ++--
 .../wal/TestCombinedAsyncWriter.java|  6 ++
 .../wal/TestSyncReplicationWALProvider.java |  6 ++
 5 files changed, 102 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/78c1d15a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index 97abc74..997a155 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,6 +25,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -220,6 +222,13 @@ public class ReplicationPeerConfig {
 return this.remoteWALDir;
   }
 
+  /**
+   * Use remote wal dir to decide whether a peer is sync replication peer
+   */
+  public boolean isSyncReplication() {
+return !StringUtils.isBlank(this.remoteWALDir);
+  }
+
   public static ReplicationPeerConfigBuilder newBuilder() {
 return new ReplicationPeerConfigBuilderImpl();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/78c1d15a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index f07a0d8..ff778a8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -170,7 +170,7 @@ public class ReplicationPeerManager {
   " does not match new remote wal dir '" + 
peerConfig.getRemoteWALDir() + "'");
 }
 
-if (oldPeerConfig.getRemoteWALDir() != null) {
+if (oldPeerConfig.isSyncReplication()) {
   if (!ReplicationUtils.isNamespacesAndTableCFsEqual(oldPeerConfig, 
peerConfig)) {
 throw new DoNotRetryIOException(
   "Changing the replicated namespace/table config on a synchronous 
replication " +
@@ -199,8 +199,8 @@ public class ReplicationPeerManager {
 }
 ReplicationPeerConfig copiedPeerConfig = 
ReplicationPeerConfig.newBuilder(peerConfig).build();
 SyncReplicationState syncReplicationState =
-StringUtils.isBlank(peerConfig.getRemoteWALDir()) ? 
SyncReplicationState.NONE
-: SyncReplicationState.DOWNGRADE_ACTIVE;
+copiedPeerConfig.isSyncReplication() ? 
SyncReplicationState.DOWNGRADE_ACTIVE
+: SyncReplicationState.NONE;
 peerStorage.addPeer(peerId, copiedPeerConfig, enabled, 
syncReplicationState);
 peers.put(peerId,
   new ReplicationPeerDescription(peerId, enabled, copiedPeerConfig, 
syncReplicationState));
@@ -324,9 +324,37 @@ public class ReplicationPeerManager {
 peerConfig.getTableCFsMap());
 }
 
+if (peerConfig.isSyncReplication()) {
+  checkPeerConfigForSyncReplication(peerConfig);
+}
+
 checkConfiguredWALEntryFilters(peerConfig);
   }
 
+  private void checkPeerConfigForSyncReplication(ReplicationPeerConfig 
peerConfig)
+  throws DoNotRetryIOException {
+// This is used to reduce the difficulty for implementing the sync 
replication state transition
+// as we need to reopen all the related regions.
+// TODO: Add namespace, replicat_all flag back
+if (peerConfig.replicateAllUserTables()) {
+  throw new DoNotRetryIOException(
+  "Only support replicated table c

[32/34] hbase git commit: HBASE-20426 Give up replicating anything in S state

[zhangduo]

HBASE-20426 Give up replicating anything in S state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/05e84437
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/05e84437
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/05e84437

Branch: refs/heads/HBASE-19064
Commit: 05e8443740e92a42fbb9c399b0d65fe8e84754a1
Parents: 94e7441
Author: zhangduo 
Authored: Thu May 3 15:51:35 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../src/main/protobuf/MasterProcedure.proto |  13 +-
 .../replication/AbstractPeerProcedure.java  |   4 +
 .../master/replication/ModifyPeerProcedure.java |   6 -
 .../replication/ReplicationPeerManager.java |  13 +-
 ...ransitPeerSyncReplicationStateProcedure.java |  94 +++
 .../hadoop/hbase/regionserver/LogRoller.java|  11 +-
 .../regionserver/PeerProcedureHandlerImpl.java  |  63 --
 .../regionserver/ReplicationSource.java |   1 +
 .../regionserver/ReplicationSourceManager.java  | 118 ---
 .../TestDrainReplicationQueuesForStandBy.java   | 118 +++
 10 files changed, 379 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/05e84437/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto 
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 01e4dae..f15cb04 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -394,11 +394,14 @@ enum PeerSyncReplicationStateTransitionState {
   SET_PEER_NEW_SYNC_REPLICATION_STATE = 2;
   REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_BEGIN = 3;
   REPLAY_REMOTE_WAL_IN_PEER = 4;
-  REOPEN_ALL_REGIONS_IN_PEER = 5;
-  TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE = 6;
-  REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END = 7;
-  CREATE_DIR_FOR_REMOTE_WAL = 8;
-  POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 9;
+  REMOVE_ALL_REPLICATION_QUEUES_IN_PEER = 5;
+  REOPEN_ALL_REGIONS_IN_PEER = 6;
+  TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE = 7;
+  REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END = 8;
+  SYNC_REPLICATION_SET_PEER_ENABLED = 9;
+  SYNC_REPLICATION_ENABLE_PEER_REFRESH_PEER_ON_RS = 10;
+  CREATE_DIR_FOR_REMOTE_WAL = 11;
+  POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 12;
 }
 
 message PeerModificationStateData {

http://git-wip-us.apache.org/repos/asf/hbase/blob/05e84437/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
index 6679d78..458e073 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
@@ -106,4 +106,8 @@ public abstract class AbstractPeerProcedure
 throw new UnsupportedOperationException();
   }
 
+  protected final void refreshPeer(MasterProcedureEnv env, PeerOperationType 
type) {
+
addChildProcedure(env.getMasterServices().getServerManager().getOnlineServersList().stream()
+  .map(sn -> new RefreshPeerProcedure(peerId, type, 
sn)).toArray(RefreshPeerProcedure[]::new));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/05e84437/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 32b8ea1..56462ca 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -108,12 +108,6 @@ public abstract class ModifyPeerProcedure extends 
AbstractPeerProcedure new RefreshPeerProcedure(peerId, type, sn))
-  .toArray(RefreshPeerProcedure[]::new));
-  }
-
   protected ReplicationPeerConfig getOldPeerConfig() {
 return null;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/05e84437/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git 
a/hbase-server/src/main/java

[17/34] hbase git commit: HBASE-19781 Add a new cluster state flag for synchronous replication

[zhangduo]

HBASE-19781 Add a new cluster state flag for synchronous replication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11a1f32c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11a1f32c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11a1f32c

Branch: refs/heads/HBASE-19064
Commit: 11a1f32c51f5434d6f205aab939fe82d456703c3
Parents: fb08423
Author: Guanghao Zhang 
Authored: Mon Jan 22 11:44:49 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../org/apache/hadoop/hbase/client/Admin.java   |  39 +
 .../apache/hadoop/hbase/client/AsyncAdmin.java  |  31 
 .../hadoop/hbase/client/AsyncHBaseAdmin.java|   7 +
 .../hbase/client/ConnectionImplementation.java  |   9 ++
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |  26 +++
 .../hadoop/hbase/client/RawAsyncHBaseAdmin.java |  15 ++
 .../client/ShortCircuitMasterConnection.java|   9 ++
 .../replication/ReplicationPeerConfigUtil.java  |  26 +--
 .../replication/ReplicationPeerDescription.java |  10 +-
 .../hbase/replication/SyncReplicationState.java |  48 ++
 .../hbase/shaded/protobuf/RequestConverter.java |  10 ++
 .../src/main/protobuf/Master.proto  |   4 +
 .../src/main/protobuf/MasterProcedure.proto |   4 +
 .../src/main/protobuf/Replication.proto |  20 +++
 .../replication/ReplicationPeerStorage.java |  18 ++-
 .../hbase/replication/ReplicationUtils.java |   1 +
 .../replication/ZKReplicationPeerStorage.java   |  61 +--
 .../replication/TestReplicationStateBasic.java  |  23 ++-
 .../TestZKReplicationPeerStorage.java   |  12 +-
 .../hbase/coprocessor/MasterObserver.java   |  23 +++
 .../org/apache/hadoop/hbase/master/HMaster.java |  12 ++
 .../hbase/master/MasterCoprocessorHost.java |  21 +++
 .../hadoop/hbase/master/MasterRpcServices.java  |  17 ++
 .../hadoop/hbase/master/MasterServices.java |   9 ++
 .../procedure/PeerProcedureInterface.java   |   2 +-
 .../replication/ReplicationPeerManager.java |  51 +-
 ...ransitPeerSyncReplicationStateProcedure.java | 159 +++
 .../hbase/security/access/AccessController.java |   8 +
 .../replication/TestReplicationAdmin.java   |  62 
 .../hbase/master/MockNoopMasterServices.java|   8 +-
 .../cleaner/TestReplicationHFileCleaner.java|   4 +-
 .../TestReplicationTrackerZKImpl.java   |   6 +-
 .../TestReplicationSourceManager.java   |   3 +-
 .../security/access/TestAccessController.java   |  16 ++
 .../hbase/util/TestHBaseFsckReplication.java|   5 +-
 .../src/main/ruby/hbase/replication_admin.rb|  15 ++
 hbase-shell/src/main/ruby/shell.rb  |   1 +
 .../src/main/ruby/shell/commands/list_peers.rb  |   6 +-
 .../transit_peer_sync_replication_state.rb  |  44 +
 .../test/ruby/hbase/replication_admin_test.rb   |  24 +++
 40 files changed, 816 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/11a1f32c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 331f2d1..39542e4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -51,6 +51,7 @@ import 
org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
@@ -2657,6 +2658,44 @@ public interface Admin extends Abortable, Closeable {
   List listReplicationPeers(Pattern pattern) 
throws IOException;
 
   /**
+   * Transit current cluster to a new state in a synchronous replication peer.
+   * @param peerId a short name that identifies the peer
+   * @param state a new state of current cluster
+   * @throws IOException if a remote or network exception occurs
+   */
+  void transitReplicationPeerSyncReplicationState(String peerId, 
SyncReplicationState state)
+  throws IOException;
+
+  /**
+   * Transit current cluster to a new state in a synchronous replication peer. 
But does not block
+   * and wait for it.
+   * 
+   * You can use Future.get(long, TimeUnit) to wait on the operation to 
complete. It may throw
+   * Exec

[22/34] hbase git commit: HBASE-19957 General framework to transit sync replication state

[zhangduo]

HBASE-19957 General framework to transit sync replication state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/61384ec6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/61384ec6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/61384ec6

Branch: refs/heads/HBASE-19064
Commit: 61384ec69b66a77086d88110d7b2f7299f38339c
Parents: 78c1d15
Author: zhangduo 
Authored: Fri Feb 9 18:33:28 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/ReplicationPeerConfig.java  |   2 -
 .../replication/ReplicationPeerDescription.java |   5 +-
 .../hbase/replication/SyncReplicationState.java |  19 +-
 .../org/apache/hadoop/hbase/HConstants.java |   3 +
 .../src/main/protobuf/MasterProcedure.proto |  20 +-
 .../hbase/replication/ReplicationPeerImpl.java  |  45 -
 .../replication/ReplicationPeerStorage.java |  25 ++-
 .../hbase/replication/ReplicationPeers.java |  27 ++-
 .../replication/ZKReplicationPeerStorage.java   |  63 +--
 .../hbase/coprocessor/MasterObserver.java   |   7 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |   4 +-
 .../hbase/master/MasterCoprocessorHost.java |  12 +-
 .../replication/AbstractPeerProcedure.java  |  14 +-
 .../master/replication/ModifyPeerProcedure.java |  11 --
 .../replication/RefreshPeerProcedure.java   |  18 +-
 .../replication/ReplicationPeerManager.java |  89 +
 ...ransitPeerSyncReplicationStateProcedure.java | 181 ---
 .../hbase/regionserver/HRegionServer.java   |  35 ++--
 .../regionserver/ReplicationSourceService.java  |  11 +-
 .../regionserver/PeerActionListener.java|   4 +-
 .../regionserver/PeerProcedureHandler.java  |  16 +-
 .../regionserver/PeerProcedureHandlerImpl.java  |  52 +-
 .../regionserver/RefreshPeerCallable.java   |   7 +
 .../replication/regionserver/Replication.java   |  22 ++-
 .../regionserver/ReplicationSourceManager.java  |  41 +++--
 .../SyncReplicationPeerInfoProvider.java|  43 +
 .../SyncReplicationPeerInfoProviderImpl.java|  71 
 .../SyncReplicationPeerMappingManager.java  |  48 +
 .../SyncReplicationPeerProvider.java|  35 
 .../hbase/wal/SyncReplicationWALProvider.java   |  35 ++--
 .../org/apache/hadoop/hbase/wal/WALFactory.java |  47 ++---
 .../replication/TestReplicationAdmin.java   |   3 +-
 .../TestReplicationSourceManager.java   |   5 +-
 .../wal/TestSyncReplicationWALProvider.java |  36 ++--
 34 files changed, 743 insertions(+), 313 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/61384ec6/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index 997a155..cc7b4bc 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -15,7 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hadoop.hbase.replication;
 
 import java.util.Collection;
@@ -25,7 +24,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.util.Bytes;

http://git-wip-us.apache.org/repos/asf/hbase/blob/61384ec6/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java
index 2d077c5..b0c27bb 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.hbase.replication;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
- * The POJO equivalent of ReplicationProtos.ReplicationPeerDescription
+ * The POJO equivalent of ReplicationProtos.ReplicationPeerDescription.
+ * 
+ * To developer, here we do not store the new sync replication state since it 
is just an
+ * intermediate state and this class is public.
  */
 @InterfaceAudience.Public
 public

[31/34] hbase git commit: HBASE-19782 Reject the replication request when peer is DA or A state

[zhangduo]

HBASE-19782 Reject the replication request when peer is DA or A state


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/428d9e29
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/428d9e29
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/428d9e29

Branch: refs/heads/HBASE-19064
Commit: 428d9e298439a361be1bb92f28c76f30b11b5914
Parents: d1727e6
Author: huzheng 
Authored: Fri Mar 2 18:05:29 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/protobuf/ReplicationProtbufUtil.java  |  2 +-
 .../hadoop/hbase/regionserver/HRegion.java  |  2 +-
 .../hbase/regionserver/HRegionServer.java   |  5 +--
 .../hbase/regionserver/RSRpcServices.java   | 25 +--
 .../RejectReplicationRequestStateChecker.java   | 45 
 .../ReplaySyncReplicationWALCallable.java   | 24 ++-
 .../replication/regionserver/Replication.java   |  2 +-
 .../regionserver/ReplicationSink.java   | 16 +++
 .../SyncReplicationPeerInfoProvider.java| 11 ++---
 .../SyncReplicationPeerInfoProviderImpl.java| 13 +++---
 .../SyncReplicationPeerMappingManager.java  |  5 +--
 .../hbase/wal/SyncReplicationWALProvider.java   |  7 +--
 .../replication/SyncReplicationTestBase.java| 32 ++
 .../replication/TestSyncReplicationActive.java  | 13 +-
 .../regionserver/TestReplicationSink.java   |  5 +--
 .../regionserver/TestWALEntrySinkFilter.java|  3 +-
 .../wal/TestSyncReplicationWALProvider.java |  6 +--
 17 files changed, 163 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/428d9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
index 81dd59e..e01f881 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.PrivateCellUtil;
+import org.apache.hadoop.hbase.wal.WAL.Entry;
 import org.apache.hadoop.hbase.wal.WALKeyImpl;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.hadoop.hbase.io.SizedCellScanner;
@@ -45,7 +46,6 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminServic
 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
 import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.wal.WAL.Entry;
 
 @InterfaceAudience.Private
 public class ReplicationProtbufUtil {

http://git-wip-us.apache.org/repos/asf/hbase/blob/428d9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 6aa4b27..ba487c6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1984,7 +1984,7 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private boolean shouldForbidMajorCompaction() {
 if (rsServices != null && rsServices.getReplicationSourceService() != 
null) {
   return 
rsServices.getReplicationSourceService().getSyncReplicationPeerInfoProvider()
-  .checkState(getRegionInfo(), ForbidMajorCompactionChecker.get());
+  .checkState(getRegionInfo().getTable(), 
ForbidMajorCompactionChecker.get());
 }
 return false;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/428d9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 440a838..ab571c6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2478,10 +2478,9 @@ public class HRegionServer extends HasThread impleme

[13/34] hbase git commit: HBASE-19999 Remove the SYNC_REPLICATION_ENABLED flag

[zhangduo]

HBASE-1 Remove the SYNC_REPLICATION_ENABLED flag


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a71ff88
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a71ff88
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a71ff88

Branch: refs/heads/HBASE-19064
Commit: 0a71ff88033efa259e993e0a66cb4f66f6c83385
Parents: 994efc3
Author: Guanghao Zhang 
Authored: Fri Mar 9 11:30:25 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/replication/ReplicationUtils.java  |  2 --
 .../hadoop/hbase/regionserver/HRegionServer.java | 13 -
 .../hbase/wal/SyncReplicationWALProvider.java| 19 ++-
 .../org/apache/hadoop/hbase/wal/WALFactory.java  | 18 --
 .../hbase/replication/TestSyncReplication.java   |  1 -
 .../master/TestRecoverStandbyProcedure.java  |  2 --
 .../wal/TestSyncReplicationWALProvider.java  |  2 --
 7 files changed, 38 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0a71ff88/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index e402d0f..cb22f57 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -37,8 +37,6 @@ import org.apache.yetus.audience.InterfaceAudience;
 @InterfaceAudience.Private
 public final class ReplicationUtils {
 
-  public static final String SYNC_REPLICATION_ENABLED = 
"hbase.replication.sync.enabled";
-
   public static final String REPLICATION_ATTR_NAME = "__rep__";
 
   public static final String REMOTE_WAL_DIR_NAME = "remoteWALs";

http://git-wip-us.apache.org/repos/asf/hbase/blob/0a71ff88/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 2fb4f67..af2f3b5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1804,10 +1804,8 @@ public class HRegionServer extends HasThread implements
   private void setupWALAndReplication() throws IOException {
 boolean isMasterNoTableOrSystemTableOnly = this instanceof HMaster &&
   (!LoadBalancer.isTablesOnMaster(conf) || 
LoadBalancer.isSystemTablesOnlyOnMaster(conf));
-if (isMasterNoTableOrSystemTableOnly) {
-  conf.setBoolean(ReplicationUtils.SYNC_REPLICATION_ENABLED, false);
-}
-WALFactory factory = new WALFactory(conf, serverName.toString());
+WALFactory factory =
+new WALFactory(conf, serverName.toString(), 
!isMasterNoTableOrSystemTableOnly);
 if (!isMasterNoTableOrSystemTableOnly) {
   // TODO Replication make assumptions here based on the default 
filesystem impl
   Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
@@ -1926,11 +1924,8 @@ public class HRegionServer extends HasThread implements
 }
 this.executorService.startExecutorService(ExecutorType.RS_REFRESH_PEER,
   conf.getInt("hbase.regionserver.executor.refresh.peer.threads", 2));
-
-if (conf.getBoolean(ReplicationUtils.SYNC_REPLICATION_ENABLED, false)) {
-  
this.executorService.startExecutorService(ExecutorType.RS_REPLAY_SYNC_REPLICATION_WAL,
-
conf.getInt("hbase.regionserver.executor.replay.sync.replication.wal.threads", 
2));
-}
+
this.executorService.startExecutorService(ExecutorType.RS_REPLAY_SYNC_REPLICATION_WAL,
+  
conf.getInt("hbase.regionserver.executor.replay.sync.replication.wal.threads", 
1));
 
 Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + 
".logRoller",
 uncaughtExceptionHandler);

http://git-wip-us.apache.org/repos/asf/hbase/blob/0a71ff88/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.java
index 282aa21..54287fe 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SyncReplicationWALProvider.jav

[06/34] hbase git commit: HBASE-20004 Client is not able to execute REST queries in a secure cluster

[zhangduo]

HBASE-20004 Client is not able to execute REST queries in a secure cluster

Signed-off-by: Ashish Singhi 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c60578d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c60578d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c60578d9

Branch: refs/heads/HBASE-19064
Commit: c60578d9829f29cf77b250d238a9e38dc0b513d7
Parents: 8ba2a7e
Author: Ashish Singhi 
Authored: Thu May 10 22:39:43 2018 +0530
Committer: Ashish Singhi 
Committed: Thu May 10 22:39:43 2018 +0530

--
 .../hadoop/hbase/http/HttpServerUtil.java   | 20 +---
 .../apache/hadoop/hbase/rest/RESTServer.java|  7 ++-
 .../hbase/rest/HBaseRESTTestingUtility.java |  2 +-
 .../hadoop/hbase/thrift/ThriftServerRunner.java |  6 +-
 4 files changed, 25 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c60578d9/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java
--
diff --git 
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java 
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java
index 777ced0..e41daf3 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java
@@ -31,8 +31,10 @@ public final class HttpServerUtil {
   /**
* Add constraints to a Jetty Context to disallow undesirable Http methods.
* @param ctxHandler The context to modify
+   * @param allowOptionsMethod if true then OPTIONS method will not be set in 
constraint mapping
*/
-  public static void constrainHttpMethods(ServletContextHandler ctxHandler) {
+  public static void constrainHttpMethods(ServletContextHandler ctxHandler,
+  boolean allowOptionsMethod) {
 Constraint c = new Constraint();
 c.setAuthenticate(true);
 
@@ -41,13 +43,17 @@ public final class HttpServerUtil {
 cmt.setMethod("TRACE");
 cmt.setPathSpec("/*");
 
-ConstraintMapping cmo = new ConstraintMapping();
-cmo.setConstraint(c);
-cmo.setMethod("OPTIONS");
-cmo.setPathSpec("/*");
-
 ConstraintSecurityHandler securityHandler = new 
ConstraintSecurityHandler();
-securityHandler.setConstraintMappings(new ConstraintMapping[]{ cmt, cmo });
+
+if (!allowOptionsMethod) {
+  ConstraintMapping cmo = new ConstraintMapping();
+  cmo.setConstraint(c);
+  cmo.setMethod("OPTIONS");
+  cmo.setPathSpec("/*");
+  securityHandler.setConstraintMappings(new ConstraintMapping[] { cmt, cmo 
});
+} else {
+  securityHandler.setConstraintMappings(new ConstraintMapping[] { cmt });
+}
 
 ctxHandler.setSecurityHandler(securityHandler);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c60578d9/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index 591eae9..cad63a4 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -95,6 +95,10 @@ public class RESTServer implements Constants {
 
   private static final String PATH_SPEC_ANY = "/*";
 
+  static String REST_HTTP_ALLOW_OPTIONS_METHOD = 
"hbase.rest.http.allow.options.method";
+  // HTTP OPTIONS method is commonly used in REST APIs for negotiation. So it 
is enabled by default.
+  private static boolean REST_HTTP_ALLOW_OPTIONS_METHOD_DEFAULT = true;
+
   private static void printUsageAndExit(Options options, int exitCode) {
 HelpFormatter formatter = new HelpFormatter();
 formatter.printHelp("hbase rest start", "", options,
@@ -341,7 +345,8 @@ public class RESTServer implements Constants {
   ctxHandler.addFilter(filter, PATH_SPEC_ANY, 
EnumSet.of(DispatcherType.REQUEST));
 }
 addCSRFFilter(ctxHandler, conf);
-HttpServerUtil.constrainHttpMethods(ctxHandler);
+HttpServerUtil.constrainHttpMethods(ctxHandler, servlet.getConfiguration()
+.getBoolean(REST_HTTP_ALLOW_OPTIONS_METHOD, 
REST_HTTP_ALLOW_OPTIONS_METHOD_DEFAULT));
 
 // Put up info server.
 int port = conf.getInt("hbase.rest.info.port", 8085);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c60578d9/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
--
diff --git 
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/HBaseRESTTestingUtility.java
 
b/hbase-rest/src/test/java/org/apache/h

[34/34] hbase git commit: HBASE-20458 Support removing a WAL from LogRoller

[zhangduo]

HBASE-20458 Support removing a WAL from LogRoller


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2ad9b55b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2ad9b55b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2ad9b55b

Branch: refs/heads/HBASE-19064
Commit: 2ad9b55be83b235613264971c0491bc1c802d16d
Parents: ccef1fa
Author: Guanghao Zhang 
Authored: Mon Apr 23 16:31:54 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hadoop/hbase/regionserver/LogRoller.java| 29 +--
 .../hbase/regionserver/wal/AbstractFSWAL.java   |  7 +-
 .../regionserver/wal/WALClosedException.java| 47 ++
 .../hbase/regionserver/TestLogRoller.java   | 90 
 .../regionserver/wal/AbstractTestFSWAL.java |  9 ++
 5 files changed, 171 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2ad9b55b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 55c5219..ab0083f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.regionserver;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -30,6 +32,7 @@ import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.regionserver.wal.WALClosedException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.wal.WAL;
@@ -177,17 +180,24 @@ public class LogRoller extends HasThread implements 
Closeable {
   rollLock.lock(); // FindBugs UL_UNRELEASED_LOCK_EXCEPTION_PATH
   try {
 this.lastrolltime = now;
-for (Entry entry : walNeedsRoll.entrySet()) {
+for (Iterator> iter = 
walNeedsRoll.entrySet().iterator(); iter
+.hasNext();) {
+  Entry entry = iter.next();
   final WAL wal = entry.getKey();
   // Force the roll if the logroll.period is elapsed or if a roll was 
requested.
   // The returned value is an array of actual region names.
-  final byte [][] regionsToFlush = wal.rollWriter(periodic ||
-  entry.getValue().booleanValue());
-  walNeedsRoll.put(wal, Boolean.FALSE);
-  if (regionsToFlush != null) {
-for (byte[] r : regionsToFlush) {
-  scheduleFlush(r);
+  try {
+final byte[][] regionsToFlush =
+wal.rollWriter(periodic || entry.getValue().booleanValue());
+walNeedsRoll.put(wal, Boolean.FALSE);
+if (regionsToFlush != null) {
+  for (byte[] r : regionsToFlush) {
+scheduleFlush(r);
+  }
 }
+  } catch (WALClosedException e) {
+LOG.warn("WAL has been closed. Skipping rolling of writer and just 
remove it", e);
+iter.remove();
   }
 }
   } catch (FailedLogCloseException e) {
@@ -252,4 +262,9 @@ public class LogRoller extends HasThread implements 
Closeable {
 running = false;
 interrupt();
   }
+
+  @VisibleForTesting
+  Map getWalNeedsRoll() {
+return this.walNeedsRoll;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2ad9b55b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index 4816d77..2c0c72b1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -750,15 +750,14 @@ public abstract class AbstractFSWAL 
implements WAL {
   public byte[][] rollWriter(boolean force) throws FailedLogCloseException, 
IOException {
 rollWriterLock.lock();
 try {
+  if (this.closed) {
+throw new WALClosedExcept

[30/34] hbase git commit: HBASE-19973 Implement a procedure to replay sync replication wal for standby cluster

[zhangduo]

HBASE-19973 Implement a procedure to replay sync replication wal for standby 
cluster


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/994efc3a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/994efc3a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/994efc3a

Branch: refs/heads/HBASE-19064
Commit: 994efc3aec83b36a1a1ddfaf93d09c3a7af9e6d5
Parents: f8f59b2
Author: Guanghao Zhang 
Authored: Fri Mar 2 18:43:25 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../src/main/protobuf/MasterProcedure.proto |  22 +++
 .../apache/hadoop/hbase/executor/EventType.java |   9 +-
 .../hadoop/hbase/executor/ExecutorType.java |   3 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |   9 +
 .../hadoop/hbase/master/MasterServices.java |   6 +
 .../procedure/PeerProcedureInterface.java   |   3 +-
 .../hbase/master/procedure/PeerQueue.java   |   3 +-
 .../replication/RecoverStandbyProcedure.java| 114 +++
 .../ReplaySyncReplicationWALManager.java| 139 +
 .../ReplaySyncReplicationWALProcedure.java  | 193 +++
 .../hbase/regionserver/HRegionServer.java   |   9 +-
 .../ReplaySyncReplicationWALCallable.java   | 149 ++
 .../SyncReplicationPeerInfoProviderImpl.java|   3 +
 .../org/apache/hadoop/hbase/util/FSUtils.java   |   5 +
 .../hbase/master/MockNoopMasterServices.java|   8 +-
 .../master/TestRecoverStandbyProcedure.java | 186 ++
 16 files changed, 854 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/994efc3a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto 
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index e8b940e..01e4dae 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -459,3 +459,25 @@ message TransitPeerSyncReplicationStateStateData {
   optional SyncReplicationState fromState = 1;
   required SyncReplicationState toState = 2;
 }
+
+enum RecoverStandbyState {
+  RENAME_SYNC_REPLICATION_WALS_DIR = 1;
+  INIT_WORKERS = 2;
+  DISPATCH_TASKS = 3;
+  REMOVE_SYNC_REPLICATION_WALS_DIR = 4;
+}
+
+message RecoverStandbyStateData {
+  required string peer_id = 1;
+}
+
+message ReplaySyncReplicationWALStateData {
+  required string peer_id = 1;
+  required string wal = 2;
+  optional ServerName target_server = 3;
+}
+
+message ReplaySyncReplicationWALParameter {
+  required string peer_id = 1;
+  required string wal = 2;
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/994efc3a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index 922deb8..ad38d1c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -281,7 +281,14 @@ public enum EventType {
*
* RS_REFRESH_PEER
*/
-  RS_REFRESH_PEER (84, ExecutorType.RS_REFRESH_PEER);
+  RS_REFRESH_PEER(84, ExecutorType.RS_REFRESH_PEER),
+
+  /**
+   * RS replay sync replication wal.
+   *
+   * RS_REPLAY_SYNC_REPLICATION_WAL
+   */
+  RS_REPLAY_SYNC_REPLICATION_WAL(85, 
ExecutorType.RS_REPLAY_SYNC_REPLICATION_WAL);
 
   private final int code;
   private final ExecutorType executor;

http://git-wip-us.apache.org/repos/asf/hbase/blob/994efc3a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index 7f130d1..ea97354 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -47,7 +47,8 @@ public enum ExecutorType {
   RS_REGION_REPLICA_FLUSH_OPS  (28),
   RS_COMPACTED_FILES_DISCHARGER (29),
   RS_OPEN_PRIORITY_REGION(30),
-  RS_REFRESH_PEER   (31);
+  RS_REFRESH_PEER(31),
+  RS_REPLAY_SYNC_REPLICATION_WAL(32);
 
   ExecutorType(int value) {
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/994efc3a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--

[09/34] hbase git commit: HBASE-19078 Add a remote peer cluster wal directory config for synchronous replication

[zhangduo]

HBASE-19078 Add a remote peer cluster wal directory config for synchronous 
replication

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cee4202a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cee4202a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cee4202a

Branch: refs/heads/HBASE-19064
Commit: cee4202aa8b24f901aad11f973978183b46c54ff
Parents: 4120d5c
Author: Guanghao Zhang 
Authored: Sat Jan 13 18:55:28 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:18:46 2018 +0800

--
 .../replication/ReplicationPeerConfigUtil.java  |  6 ++
 .../replication/ReplicationPeerConfig.java  | 20 -
 .../ReplicationPeerConfigBuilder.java   |  7 ++
 .../src/main/protobuf/Replication.proto |  1 +
 .../replication/ReplicationPeerManager.java | 15 
 .../replication/TestReplicationAdmin.java   | 77 
 .../src/main/ruby/hbase/replication_admin.rb| 14 ++--
 hbase-shell/src/main/ruby/hbase_constants.rb|  1 +
 .../src/main/ruby/shell/commands/add_peer.rb| 21 +-
 .../src/main/ruby/shell/commands/list_peers.rb  | 19 -
 .../test/ruby/hbase/replication_admin_test.rb   | 16 
 11 files changed, 186 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cee4202a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index b1c1713..474ded3 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -319,6 +319,9 @@ public final class ReplicationPeerConfigUtil {
 
excludeNamespacesList.stream().map(ByteString::toStringUtf8).collect(Collectors.toSet()));
 }
 
+if (peer.hasRemoteWALDir()) {
+  builder.setRemoteWALDir(peer.getRemoteWALDir());
+}
 return builder.build();
   }
 
@@ -376,6 +379,9 @@ public final class ReplicationPeerConfigUtil {
   }
 }
 
+if (peerConfig.getRemoteWALDir() != null) {
+  builder.setRemoteWALDir(peerConfig.getRemoteWALDir());
+}
 return builder.build();
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/cee4202a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index e0d9a4c..97abc74 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -47,6 +47,8 @@ public class ReplicationPeerConfig {
   private Set excludeNamespaces = null;
   private long bandwidth = 0;
   private final boolean serial;
+  // Used by synchronous replication
+  private String remoteWALDir;
 
   private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) {
 this.clusterKey = builder.clusterKey;
@@ -66,6 +68,7 @@ public class ReplicationPeerConfig {
 : null;
 this.bandwidth = builder.bandwidth;
 this.serial = builder.serial;
+this.remoteWALDir = builder.remoteWALDir;
   }
 
   private Map>
@@ -213,6 +216,10 @@ public class ReplicationPeerConfig {
 return this;
   }
 
+  public String getRemoteWALDir() {
+return this.remoteWALDir;
+  }
+
   public static ReplicationPeerConfigBuilder newBuilder() {
 return new ReplicationPeerConfigBuilderImpl();
   }
@@ -230,7 +237,8 @@ public class ReplicationPeerConfig {
   .setReplicateAllUserTables(peerConfig.replicateAllUserTables())
   .setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap())
   .setExcludeNamespaces(peerConfig.getExcludeNamespaces())
-  
.setBandwidth(peerConfig.getBandwidth()).setSerial(peerConfig.isSerial());
+  .setBandwidth(peerConfig.getBandwidth()).setSerial(peerConfig.isSerial())
+  .setRemoteWALDir(peerConfig.getRemoteWALDir());
 return builder;
   }
 
@@ -259,6 +267,8 @@ public class ReplicationPeerConfig {
 
 private boolean serial = false;
 
+private String remoteWALDir = null;
+
 @Override
 public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) {
   this.clusterKey = clusterKey;
@@ -327,6 +337,11 @@ public

[03/34] hbase git commit: HBASE-20554 "WALs outstanding" message from CleanerChore is noisy

[zhangduo]

HBASE-20554 "WALs outstanding" message from CleanerChore is noisy


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/32dd633b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/32dd633b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/32dd633b

Branch: refs/heads/HBASE-19064
Commit: 32dd633b9899a2cae70c017c33a616701d6cf7bc
Parents: 9c046c0
Author: Andrew Purtell 
Authored: Wed May 9 10:16:44 2018 -0700
Committer: Andrew Purtell 
Committed: Wed May 9 19:08:35 2018 -0700

--
 .../hadoop/hbase/master/cleaner/CleanerChore.java   | 12 +---
 1 file changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/32dd633b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index dc13645..bcb23ac 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -272,9 +272,13 @@ public abstract class CleanerChore extends Schedu
   try {
 POOL.latchCountUp();
 if (runCleaner()) {
-  LOG.debug("Cleaned all WALs under {}", oldFileDir);
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Cleaned all WALs under {}", oldFileDir);
+  }
 } else {
-  LOG.warn("WALs outstanding under {}", oldFileDir);
+  if (LOG.isTraceEnabled()) {
+LOG.trace("WALs outstanding under {}", oldFileDir);
+  }
 }
   } finally {
 POOL.latchCountDown();
@@ -287,7 +291,9 @@ public abstract class CleanerChore extends Schedu
 POOL.updatePool((long) (0.8 * getTimeUnit().toMillis(getPeriod(;
   }
 } else {
-  LOG.debug("Cleaner chore disabled! Not cleaning.");
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Cleaner chore disabled! Not cleaning.");
+  }
 }
   }
 

[21/34] hbase git commit: HBASE-19957 General framework to transit sync replication state

[zhangduo]

http://git-wip-us.apache.org/repos/asf/hbase/blob/61384ec6/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProvider.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProvider.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProvider.java
new file mode 100644
index 000..92f2c52
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProvider.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import java.util.Optional;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Get the information for a sync replication peer.
+ */
+@InterfaceAudience.Private
+public interface SyncReplicationPeerInfoProvider {
+
+  /**
+   * Return the peer id and remote WAL directory if the region is 
synchronously replicated and the
+   * state is {@link SyncReplicationState#ACTIVE}.
+   */
+  Optional> getPeerIdAndRemoteWALDir(RegionInfo info);
+
+  /**
+   * Check whether the give region is contained in a sync replication peer 
which is in the given
+   * state.
+   */
+  boolean isInState(RegionInfo info, SyncReplicationState state);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/61384ec6/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProviderImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProviderImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProviderImpl.java
new file mode 100644
index 000..32159e6
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SyncReplicationPeerInfoProviderImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import java.util.Optional;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+class SyncReplicationPeerInfoProviderImpl implements 
SyncReplicationPeerInfoProvider {
+
+  private final ReplicationPeers replicationPeers;
+
+  private final SyncReplicationPeerMappingManager mapping;
+
+  SyncReplicationPeerInfoProviderImpl(ReplicationPeers replicationPeers,
+  SyncReplicationPeerMappingManager mapping) {
+this.replicationPeers = replicationPeers;
+this.mapping = mapping;
+  }
+
+  @Override
+  public Optional> getPeerIdAndRemoteWALDir(RegionInfo 
info) {
+String peerId = mapping.getPeerId(info);
+if (peerId == null) {
+  return Optional.empty();
+}
+ReplicationPeer peer = replication

[25/34] hbase git commit: HBASE-20456 Support removing a ReplicationSourceShipper for a special wal group

[zhangduo]

HBASE-20456 Support removing a ReplicationSourceShipper for a special wal group


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/39709478
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/39709478
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/39709478

Branch: refs/heads/HBASE-19064
Commit: 39709478e2abb3f5d188e98c91804f1fac8a7b07
Parents: 6265431
Author: zhangduo 
Authored: Tue Apr 24 22:01:21 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/regionserver/wal/AsyncFSWAL.java  |  1 +
 .../RecoveredReplicationSource.java | 13 +---
 .../RecoveredReplicationSourceShipper.java  |  7 --
 .../regionserver/ReplicationSource.java | 13 +++-
 .../regionserver/ReplicationSourceManager.java  | 19 -
 .../regionserver/ReplicationSourceShipper.java  | 20 +++--
 .../ReplicationSourceWALReader.java |  9 ++-
 .../regionserver/WALEntryStream.java|  3 +-
 .../hadoop/hbase/wal/AbstractFSWALProvider.java | 28 ---
 .../hbase/wal/SyncReplicationWALProvider.java   | 10 ++-
 .../TestReplicationSourceManager.java   |  5 +-
 .../TestSyncReplicationShipperQuit.java | 81 
 .../regionserver/TestWALEntryStream.java|  4 +-
 13 files changed, 163 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/39709478/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 17133ed..f630e63 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -682,6 +682,7 @@ public class AsyncFSWAL extends AbstractFSWAL {
   protected void doShutdown() throws IOException {
 waitForSafePoint();
 closeWriter(this.writer);
+this.writer = null;
 closeExecutor.shutdown();
 try {
   if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds, 
TimeUnit.SECONDS)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/39709478/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index a21ca44..f1bb538 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.replication.ReplicationPeer;
 import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
@@ -144,15 +143,9 @@ public class RecoveredReplicationSource extends 
ReplicationSource {
   }
 
   void tryFinish() {
-// use synchronize to make sure one last thread will clean the queue
-synchronized (workerThreads) {
-  Threads.sleep(100);// wait a short while for other worker thread to 
fully exit
-  boolean allTasksDone = workerThreads.values().stream().allMatch(w -> 
w.isFinished());
-  if (allTasksDone) {
-this.getSourceMetrics().clear();
-manager.removeRecoveredSource(this);
-LOG.info("Finished recovering queue {} with the following stats: {}", 
queueId, getStats());
-  }
+if (workerThreads.isEmpty()) {
+  this.getSourceMetrics().clear();
+  manager.finishRecoveredSource(this);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/39709478/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
index 91109cf..b0d4db0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/re

[10/34] hbase git commit: HBASE-19083 Introduce a new log writer which can write to two HDFSes

[zhangduo]

HBASE-19083 Introduce a new log writer which can write to two HDFSes


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4120d5c4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4120d5c4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4120d5c4

Branch: refs/heads/HBASE-19064
Commit: 4120d5c4b6a542839538e6e061b525219ac94263
Parents: 021f66d
Author: zhangduo 
Authored: Thu Jan 11 21:08:02 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:18:46 2018 +0800

--
 .../hbase/regionserver/wal/AsyncFSWAL.java  |  21 +--
 .../regionserver/wal/CombinedAsyncWriter.java   | 134 ++
 .../hbase/regionserver/wal/DualAsyncFSWAL.java  |  67 +
 .../wal/AbstractTestProtobufLog.java| 110 +++
 .../regionserver/wal/ProtobufLogTestHelper.java |  99 ++
 .../regionserver/wal/TestAsyncProtobufLog.java  |  32 +
 .../wal/TestCombinedAsyncWriter.java| 136 +++
 .../hbase/regionserver/wal/TestProtobufLog.java |  14 +-
 .../regionserver/wal/WriterOverAsyncWriter.java |  63 +
 9 files changed, 533 insertions(+), 143 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4120d5c4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index e34818f..0bee9d6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -607,12 +607,16 @@ public class AsyncFSWAL extends 
AbstractFSWAL {
 }
   }
 
-  @Override
-  protected AsyncWriter createWriterInstance(Path path) throws IOException {
+  protected final AsyncWriter createAsyncWriter(FileSystem fs, Path path) 
throws IOException {
 return AsyncFSWALProvider.createAsyncWriter(conf, fs, path, false, 
eventLoopGroup,
   channelClass);
   }
 
+  @Override
+  protected AsyncWriter createWriterInstance(Path path) throws IOException {
+return createAsyncWriter(fs, path);
+  }
+
   private void waitForSafePoint() {
 consumeLock.lock();
 try {
@@ -632,13 +636,12 @@ public class AsyncFSWAL extends 
AbstractFSWAL {
 }
   }
 
-  private long closeWriter() {
-AsyncWriter oldWriter = this.writer;
-if (oldWriter != null) {
-  long fileLength = oldWriter.getLength();
+  protected final long closeWriter(AsyncWriter writer) {
+if (writer != null) {
+  long fileLength = writer.getLength();
   closeExecutor.execute(() -> {
 try {
-  oldWriter.close();
+  writer.close();
 } catch (IOException e) {
   LOG.warn("close old writer failed", e);
 }
@@ -654,7 +657,7 @@ public class AsyncFSWAL extends AbstractFSWAL {
   throws IOException {
 Preconditions.checkNotNull(nextWriter);
 waitForSafePoint();
-long oldFileLen = closeWriter();
+long oldFileLen = closeWriter(this.writer);
 logRollAndSetupWalProps(oldPath, newPath, oldFileLen);
 this.writer = nextWriter;
 if (nextWriter instanceof AsyncProtobufLogWriter) {
@@ -679,7 +682,7 @@ public class AsyncFSWAL extends AbstractFSWAL {
   @Override
   protected void doShutdown() throws IOException {
 waitForSafePoint();
-closeWriter();
+closeWriter(this.writer);
 closeExecutor.shutdown();
 try {
   if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds, 
TimeUnit.SECONDS)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/4120d5c4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
new file mode 100644
index 000..8ecfede
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by app

[12/34] hbase git commit: HBASE-20425 Do not write the cluster id of the current active cluster when writing remote WAL

[zhangduo]

HBASE-20425 Do not write the cluster id of the current active cluster when 
writing remote WAL


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/62654313
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/62654313
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/62654313

Branch: refs/heads/HBASE-19064
Commit: 6265431338afe00a1c9c76c479b3448cd1c69f03
Parents: 428d9e2
Author: huzheng 
Authored: Mon Apr 23 17:20:55 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/TestSyncReplicationActive.java  | 32 
 1 file changed, 32 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/62654313/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSyncReplicationActive.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSyncReplicationActive.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSyncReplicationActive.java
index bff4572..f9020a0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSyncReplicationActive.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSyncReplicationActive.java
@@ -17,9 +17,17 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.wal.WAL.Entry;
+import org.apache.hadoop.hbase.wal.WAL.Reader;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.junit.Assert;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -49,6 +57,9 @@ public class TestSyncReplicationActive extends 
SyncReplicationTestBase {
 // peer is disabled so no data have been replicated
 verifyNotReplicatedThroughRegion(UTIL2, 0, 100);
 
+// Ensure that there's no cluster id in remote log entries.
+verifyNoClusterIdInRemoteLog(UTIL2, PEER_ID);
+
 UTIL2.getAdmin().transitReplicationPeerSyncReplicationState(PEER_ID,
   SyncReplicationState.DOWNGRADE_ACTIVE);
 // confirm that peer with state DA will reject replication request.
@@ -72,4 +83,25 @@ public class TestSyncReplicationActive extends 
SyncReplicationTestBase {
 verifyReplicationRequestRejection(UTIL2, true);
 write(UTIL2, 200, 300);
   }
+
+  private void verifyNoClusterIdInRemoteLog(HBaseTestingUtility utility, 
String peerId)
+  throws Exception {
+FileSystem fs2 = utility.getTestFileSystem();
+Path remoteDir =
+new 
Path(utility.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir(),
+"remoteWALs").makeQualified(fs2.getUri(), 
fs2.getWorkingDirectory());
+FileStatus[] files = fs2.listStatus(new Path(remoteDir, peerId));
+Assert.assertTrue(files.length > 0);
+for (FileStatus file : files) {
+  try (Reader reader =
+  WALFactory.createReader(fs2, file.getPath(), 
utility.getConfiguration())) {
+Entry entry = reader.next();
+Assert.assertTrue(entry != null);
+while (entry != null) {
+  Assert.assertEquals(entry.getKey().getClusterIds().size(), 0);
+  entry = reader.next();
+}
+  }
+}
+  }
 }

[04/34] hbase git commit: HBASE-20545 Improve performance of BaseLoadBalancer.retainAssignment

[zhangduo]

HBASE-20545 Improve performance of BaseLoadBalancer.retainAssignment

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a67909d3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a67909d3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a67909d3

Branch: refs/heads/HBASE-19064
Commit: a67909d3d64675f5f557e499cdcc8898fd33e31b
Parents: 32dd633
Author: Thiruvel Thirumoolan 
Authored: Wed May 9 11:47:48 2018 -0700
Committer: tedyu 
Committed: Wed May 9 19:48:27 2018 -0700

--
 .../hbase/master/balancer/BaseLoadBalancer.java | 41 ++--
 1 file changed, 29 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a67909d3/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 44dfe41..90f4e58 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -1413,6 +1413,9 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 // after the cluster restart.
 Set oldHostsNoLongerPresent = Sets.newTreeSet();
 
+// If the old servers aren't present, lets assign those regions later.
+List randomAssignRegions = Lists.newArrayList();
+
 int numRandomAssignments = 0;
 int numRetainedAssigments = 0;
 
@@ -1426,37 +1429,51 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 localServers = serversByHostname.get(oldServerName.getHostname());
   }
   if (localServers.isEmpty()) {
-// No servers on the new cluster match up with this hostname,
-// assign randomly.
-ServerName randomServer = randomAssignment(cluster, region, servers);
-assignments.get(randomServer).add(region);
-numRandomAssignments++;
-if (oldServerName != null) 
oldHostsNoLongerPresent.add(oldServerName.getHostname());
+// No servers on the new cluster match up with this hostname, assign 
randomly, later.
+randomAssignRegions.add(region);
+if (oldServerName != null) {
+  oldHostsNoLongerPresent.add(oldServerName.getHostname());
+}
   } else if (localServers.size() == 1) {
 // the usual case - one new server on same host
 ServerName target = localServers.get(0);
 assignments.get(target).add(region);
-cluster.doAssignRegion(region, target);
 numRetainedAssigments++;
   } else {
 // multiple new servers in the cluster on this same host
 if (localServers.contains(oldServerName)) {
   assignments.get(oldServerName).add(region);
-  cluster.doAssignRegion(region, oldServerName);
+  numRetainedAssigments++;
 } else {
   ServerName target = null;
-  for (ServerName tmp: localServers) {
+  for (ServerName tmp : localServers) {
 if (tmp.getPort() == oldServerName.getPort()) {
   target = tmp;
+  assignments.get(tmp).add(region);
+  numRetainedAssigments++;
   break;
 }
   }
   if (target == null) {
-target = randomAssignment(cluster, region, localServers);
+randomAssignRegions.add(region);
   }
-  assignments.get(target).add(region);
 }
-numRetainedAssigments++;
+  }
+}
+
+// If servers from prior assignment aren't present, then lets do 
randomAssignment on regions.
+if (randomAssignRegions.size() > 0) {
+  for (Map.Entry> entry : 
assignments.entrySet()) {
+ServerName sn = entry.getKey();
+for (RegionInfo region : entry.getValue()) {
+  cluster.doAssignRegion(region, sn);
+}
+  }
+  for (RegionInfo region : randomAssignRegions) {
+ServerName target = randomAssignment(cluster, region, servers);
+assignments.get(target).add(region);
+cluster.doAssignRegion(region, target);
+numRandomAssignments++;
   }
 }
 

[15/34] hbase git commit: HBASE-19747 Introduce a special WALProvider for synchronous replication

[zhangduo]

HBASE-19747 Introduce a special WALProvider for synchronous replication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fb084231
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fb084231
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fb084231

Branch: refs/heads/HBASE-19064
Commit: fb084231e62952205fc7dde1cd86f45ce40188a4
Parents: cee4202
Author: zhangduo 
Authored: Fri Jan 19 18:38:39 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/regionserver/wal/AbstractFSWAL.java   |   7 +
 .../hbase/regionserver/wal/AsyncFSWAL.java  |   1 -
 .../hbase/regionserver/wal/DualAsyncFSWAL.java  |   4 +-
 .../hadoop/hbase/regionserver/wal/FSHLog.java   |   4 -
 .../regionserver/PeerActionListener.java|  33 +++
 .../SynchronousReplicationPeerProvider.java |  35 +++
 .../hadoop/hbase/wal/AbstractFSWALProvider.java |   1 +
 .../hadoop/hbase/wal/AsyncFSWALProvider.java|  18 +-
 .../hbase/wal/NettyAsyncFSWALConfigHelper.java  |   8 +-
 .../hbase/wal/RegionGroupingProvider.java   |  13 +-
 .../wal/SynchronousReplicationWALProvider.java  | 225 +++
 .../org/apache/hadoop/hbase/wal/WALFactory.java |  37 ++-
 .../org/apache/hadoop/hbase/wal/WALKeyImpl.java |  16 +-
 .../regionserver/TestCompactionPolicy.java  |   1 +
 .../regionserver/TestFailedAppendAndSync.java   | 122 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |  24 +-
 .../TestHRegionWithInMemoryFlush.java   |   7 -
 .../hbase/regionserver/TestRegionIncrement.java |  20 +-
 .../hbase/regionserver/TestWALLockup.java   |   1 +
 .../regionserver/wal/AbstractTestWALReplay.java |   1 +
 .../regionserver/wal/ProtobufLogTestHelper.java |  44 +++-
 .../hbase/regionserver/wal/TestAsyncFSWAL.java  |  13 +-
 .../regionserver/wal/TestAsyncWALReplay.java|   4 +-
 .../wal/TestCombinedAsyncWriter.java|   3 +-
 .../hbase/regionserver/wal/TestFSHLog.java  |  15 +-
 .../hbase/regionserver/wal/TestWALReplay.java   |   1 +
 .../apache/hadoop/hbase/wal/IOTestProvider.java |   2 -
 .../TestSynchronousReplicationWALProvider.java  | 153 +
 28 files changed, 659 insertions(+), 154 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fb084231/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index ce8dafa..4816d77 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -430,6 +430,13 @@ public abstract class AbstractFSWAL 
implements WAL {
 this.implClassName = getClass().getSimpleName();
   }
 
+  /**
+   * Used to initialize the WAL. Usually just call rollWriter to create the 
first log writer.
+   */
+  public void init() throws IOException {
+rollWriter();
+  }
+
   @Override
   public void registerWALActionsListener(WALActionsListener listener) {
 this.listeners.add(listener);

http://git-wip-us.apache.org/repos/asf/hbase/blob/fb084231/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 0bee9d6..17133ed 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -248,7 +248,6 @@ public class AsyncFSWAL extends AbstractFSWAL {
 batchSize = conf.getLong(WAL_BATCH_SIZE, DEFAULT_WAL_BATCH_SIZE);
 waitOnShutdownInSeconds = 
conf.getInt(ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS,
   DEFAULT_ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS);
-rollWriter();
   }
 
   private static boolean waitingRoll(int epochAndState) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/fb084231/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
index 42b0dae..0495337 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
+++ 
b/hbase-ser

[18/34] hbase git commit: HBASE-19857 Complete the procedure for adding a sync replication peer

[zhangduo]

HBASE-19857 Complete the procedure for adding a sync replication peer


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b10314b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b10314b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b10314b0

Branch: refs/heads/HBASE-19064
Commit: b10314b000b7a6eb7afb3991469a70d10dea6586
Parents: 11a1f32
Author: zhangduo 
Authored: Thu Jan 25 20:09:00 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/replication/ReplicationPeer.java  |   9 +
 .../hbase/replication/ReplicationPeerImpl.java  |  28 +--
 .../hbase/replication/ReplicationPeers.java |   3 +-
 .../regionserver/PeerActionListener.java|  10 +-
 .../SyncReplicationPeerProvider.java|  35 +++
 .../SynchronousReplicationPeerProvider.java |  35 ---
 .../hbase/wal/SyncReplicationWALProvider.java   | 234 +++
 .../wal/SynchronousReplicationWALProvider.java  | 225 --
 .../org/apache/hadoop/hbase/wal/WALFactory.java |   8 +-
 .../TestReplicationSourceManager.java   |   3 +
 .../wal/TestSyncReplicationWALProvider.java | 153 
 .../TestSynchronousReplicationWALProvider.java  | 153 
 12 files changed, 456 insertions(+), 440 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b10314b0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 2da3cce..0196a9a 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,15 @@ public interface ReplicationPeer {
   PeerState getPeerState();
 
   /**
+   * Returns the sync replication state of the peer by reading local cache.
+   * 
+   * If the peer is not a synchronous replication peer, a {@link 
SyncReplicationState#NONE} will be
+   * returned.
+   * @return the sync replication state
+   */
+  SyncReplicationState getSyncReplicationState();
+
+  /**
* Test whether the peer is enabled.
* @return {@code true} if enabled, otherwise {@code false}.
*/

http://git-wip-us.apache.org/repos/asf/hbase/blob/b10314b0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index d656466..ff3f662 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -36,6 +36,8 @@ public class ReplicationPeerImpl implements ReplicationPeer {
 
   private volatile PeerState peerState;
 
+  private volatile SyncReplicationState syncReplicationState;
+
   private final List peerConfigListeners;
 
   /**
@@ -45,12 +47,13 @@ public class ReplicationPeerImpl implements ReplicationPeer 
{
* @param id string representation of this peer's identifier
* @param peerConfig configuration for the replication peer
*/
-  public ReplicationPeerImpl(Configuration conf, String id, boolean peerState,
-  ReplicationPeerConfig peerConfig) {
+  public ReplicationPeerImpl(Configuration conf, String id, 
ReplicationPeerConfig peerConfig,
+  boolean peerState, SyncReplicationState syncReplicationState) {
 this.conf = conf;
 this.id = id;
 this.peerState = peerState ? PeerState.ENABLED : PeerState.DISABLED;
 this.peerConfig = peerConfig;
+this.syncReplicationState = syncReplicationState;
 this.peerConfigListeners = new ArrayList<>();
   }
 
@@ -77,37 +80,26 @@ public class ReplicationPeerImpl implements ReplicationPeer 
{
 return peerState;
   }
 
-  /**
-   * Get the peer config object
-   * @return the ReplicationPeerConfig for this peer
-   */
+  @Override
+  public SyncReplicationState getSyncReplicationState() {
+return syncReplicationState;
+  }
+
   @Override
   public ReplicationPeerConfig getPeerConfig() {
 return peerConfig;
   }
 
-  /**
-   * Get the configuration object required to communicate with this peer
-   * @return configuration object
-   */
   @Override
   public Configuration getConfiguration() {
 return conf;
   }
 
-  /**
-   * Get replicable (table, cf-list) map of

[28/34] hbase git commit: HBASE-20163 Forbid major compaction when standby cluster replay the remote wals

[zhangduo]

HBASE-20163 Forbid major compaction when standby cluster replay the remote wals


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0588a763
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0588a763
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0588a763

Branch: refs/heads/HBASE-19064
Commit: 0588a763f22c9150a5011967f104d69cd654a556
Parents: 010fa53
Author: Guanghao Zhang 
Authored: Thu Apr 12 14:44:25 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  | 18 
 .../hbase/regionserver/HRegionServer.java   |  2 +-
 .../regionserver/RegionServerServices.java  |  5 +++
 .../ForbidMajorCompactionChecker.java   | 44 
 .../hadoop/hbase/MockRegionServerServices.java  |  6 +++
 .../hadoop/hbase/master/MockRegionServer.java   |  6 +++
 6 files changed, 80 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0588a763/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index d86565e..6aa4b27 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -144,6 +144,7 @@ import 
org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
 import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import 
org.apache.hadoop.hbase.regionserver.compactions.ForbidMajorCompactionChecker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
 import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
@@ -1980,6 +1981,14 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 return compact(compaction, store, throughputController, null);
   }
 
+  private boolean shouldForbidMajorCompaction() {
+if (rsServices != null && rsServices.getReplicationSourceService() != 
null) {
+  return 
rsServices.getReplicationSourceService().getSyncReplicationPeerInfoProvider()
+  .checkState(getRegionInfo(), ForbidMajorCompactionChecker.get());
+}
+return false;
+  }
+
   public boolean compact(CompactionContext compaction, HStore store,
   ThroughputController throughputController, User user) throws IOException 
{
 assert compaction != null && compaction.hasSelection();
@@ -1989,6 +1998,15 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   store.cancelRequestedCompaction(compaction);
   return false;
 }
+
+if (compaction.getRequest().isAllFiles() && shouldForbidMajorCompaction()) 
{
+  LOG.warn("Skipping major compaction on " + this
+  + " because this cluster is transiting sync replication state"
+  + " from STANDBY to DOWNGRADE_ACTIVE");
+  store.cancelRequestedCompaction(compaction);
+  return false;
+}
+
 MonitoredTask status = null;
 boolean requestNeedsCancellation = true;
 /*

http://git-wip-us.apache.org/repos/asf/hbase/blob/0588a763/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index af2f3b5..440a838 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2472,7 +2472,7 @@ public class HRegionServer extends HasThread implements
* @return Return the object that implements the replication
* source executorService.
*/
-  @VisibleForTesting
+  @Override
   public ReplicationSourceService getReplicationSourceService() {
 return replicationSourceHandler;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0588a763/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
 
b/hbase-server

[01/34] hbase git commit: HBASE-20539 Disable IMC; part 2 [Forced Update!]

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-19064 3f08903e8 -> 0567fbb4b (forced update)


HBASE-20539 Disable IMC; part 2

Set default for in-memory compaction to NONE. PE was using
this value creating its table. Add test to ensure expected
default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4f2dfd38
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4f2dfd38
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4f2dfd38

Branch: refs/heads/HBASE-19064
Commit: 4f2dfd387779968d3b76a22f1f4189d3278b202d
Parents: f8281aa
Author: Michael Stack 
Authored: Tue May 8 10:08:40 2018 -0700
Committer: Michael Stack 
Committed: Wed May 9 06:30:28 2018 -0700

--
 .../hadoop/hbase/TestPerformanceEvaluation.java   | 14 ++
 .../hadoop/hbase/regionserver/CompactingMemStore.java |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4f2dfd38/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
--
diff --git 
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
 
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index ab93857..b2a89c8 100644
--- 
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++ 
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.PerformanceEvaluation.RandomReadTest;
 import org.apache.hadoop.hbase.PerformanceEvaluation.TestOptions;
+import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.ClassRule;
@@ -57,6 +58,19 @@ public class TestPerformanceEvaluation {
   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
 
   @Test
+  public void testDefaultInMemoryCompaction() {
+PerformanceEvaluation.TestOptions defaultOpts =
+new PerformanceEvaluation.TestOptions();
+
assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT.toString(),
+defaultOpts.getInMemoryCompaction().toString());
+HTableDescriptor htd = 
PerformanceEvaluation.getTableDescriptor(defaultOpts);
+for (HColumnDescriptor hcd: htd.getFamilies()) {
+  
assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT.toString(),
+  hcd.getInMemoryCompaction().toString());
+}
+  }
+
+  @Test
   public void testSerialization()
   throws JsonGenerationException, JsonMappingException, IOException {
 PerformanceEvaluation.TestOptions options = new 
PerformanceEvaluation.TestOptions();

http://git-wip-us.apache.org/repos/asf/hbase/blob/4f2dfd38/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 68d7d7e..c1b061a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -58,7 +58,7 @@ public class CompactingMemStore extends AbstractMemStore {
   public static final String COMPACTING_MEMSTORE_TYPE_KEY =
   "hbase.hregion.compacting.memstore.type";
   public static final String COMPACTING_MEMSTORE_TYPE_DEFAULT =
-  String.valueOf(MemoryCompactionPolicy.BASIC);
+  String.valueOf(MemoryCompactionPolicy.NONE);
   // Default fraction of in-memory-flush size w.r.t. flush-to-disk size
   public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY =
   "hbase.memstore.inmemoryflush.threshold.factor";

[14/34] hbase git commit: HBASE-19079 Support setting up two clusters with A and S stat

[zhangduo]

HBASE-19079 Support setting up two clusters with A and S stat


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/010fa530
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/010fa530
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/010fa530

Branch: refs/heads/HBASE-19064
Commit: 010fa530d5c29b0fee036e721a13a41b8d44182a
Parents: 0a71ff8
Author: zhangduo 
Authored: Tue Apr 10 22:35:19 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/ReplicationPeerManager.java |   5 +-
 ...ransitPeerSyncReplicationStateProcedure.java |   2 +-
 .../hbase/regionserver/wal/DualAsyncFSWAL.java  |  14 ++
 .../hadoop/hbase/regionserver/wal/WALUtil.java  |  25 ++-
 .../hbase/replication/ChainWALEntryFilter.java  |  28 +--
 .../ReplaySyncReplicationWALCallable.java   |  27 ++-
 .../SyncReplicationPeerInfoProviderImpl.java|   6 +-
 .../hadoop/hbase/wal/AbstractFSWALProvider.java |  10 +-
 .../hbase/wal/SyncReplicationWALProvider.java   |  94 ++---
 .../org/apache/hadoop/hbase/wal/WALEdit.java|   8 +-
 .../org/apache/hadoop/hbase/wal/WALFactory.java |   2 +-
 .../replication/TestReplicationAdmin.java   |  33 +--
 .../regionserver/wal/TestWALDurability.java |   2 +
 .../replication/SyncReplicationTestBase.java| 185 +
 .../hbase/replication/TestSyncReplication.java  | 207 ---
 .../replication/TestSyncReplicationActive.java  |  64 ++
 .../replication/TestSyncReplicationStandBy.java |  96 +
 17 files changed, 521 insertions(+), 287 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/010fa530/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 41dd6e3..229549e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -68,8 +68,9 @@ public class ReplicationPeerManager {
 
   private final ImmutableMap>
 allowedTransition = 
Maps.immutableEnumMap(ImmutableMap.of(SyncReplicationState.ACTIVE,
-  EnumSet.of(SyncReplicationState.DOWNGRADE_ACTIVE), 
SyncReplicationState.STANDBY,
-  EnumSet.of(SyncReplicationState.DOWNGRADE_ACTIVE), 
SyncReplicationState.DOWNGRADE_ACTIVE,
+  EnumSet.of(SyncReplicationState.DOWNGRADE_ACTIVE, 
SyncReplicationState.STANDBY),
+  SyncReplicationState.STANDBY, 
EnumSet.of(SyncReplicationState.DOWNGRADE_ACTIVE),
+  SyncReplicationState.DOWNGRADE_ACTIVE,
   EnumSet.of(SyncReplicationState.STANDBY, SyncReplicationState.ACTIVE)));
 
   ReplicationPeerManager(ReplicationPeerStorage peerStorage, 
ReplicationQueueStorage queueStorage,

http://git-wip-us.apache.org/repos/asf/hbase/blob/010fa530/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
index cc51890..5da2b0c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
@@ -171,7 +171,7 @@ public class TransitPeerSyncReplicationStateProcedure
 }
 return Flow.HAS_MORE_STATE;
   case REPLAY_REMOTE_WAL_IN_PEER:
-// TODO: replay remote wal when transiting from S to DA.
+addChildProcedure(new RecoverStandbyProcedure(peerId));
 
setNextState(PeerSyncReplicationStateTransitionState.REOPEN_ALL_REGIONS_IN_PEER);
 return Flow.HAS_MORE_STATE;
   case REOPEN_ALL_REGIONS_IN_PEER:

http://git-wip-us.apache.org/repos/asf/hbase/blob/010fa530/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
index 0495337..a98567a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyn

[26/34] hbase git commit: HBASE-20432 Cleanup related resources when remove a sync replication peer

[zhangduo]

HBASE-20432 Cleanup related resources when remove a sync replication peer


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94e74419
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94e74419
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94e74419

Branch: refs/heads/HBASE-19064
Commit: 94e74419506a5f5642c5c35489b8e43e79ccf1d4
Parents: 2ad9b55
Author: huzheng 
Authored: Wed Apr 18 20:38:33 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../master/replication/RemovePeerProcedure.java | 10 +
 .../ReplaySyncReplicationWALManager.java|  8 
 .../replication/SyncReplicationTestBase.java| 45 +---
 .../replication/TestSyncReplicationActive.java  |  9 ++--
 .../replication/TestSyncReplicationStandBy.java | 31 --
 5 files changed, 89 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/94e74419/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
index 82dc07e..7335fe0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -66,9 +66,19 @@ public class RemovePeerProcedure extends ModifyPeerProcedure 
{
 env.getReplicationPeerManager().removePeer(peerId);
   }
 
+  private void removeRemoteWALs(MasterProcedureEnv env) throws IOException {
+ReplaySyncReplicationWALManager remoteWALManager =
+env.getMasterServices().getReplaySyncReplicationWALManager();
+remoteWALManager.removePeerRemoteWALs(peerId);
+remoteWALManager.removePeerReplayWALDir(peerId);
+  }
+
   @Override
   protected void postPeerModification(MasterProcedureEnv env)
   throws IOException, ReplicationException {
+if (peerConfig.isSyncReplication()) {
+  removeRemoteWALs(env);
+}
 env.getReplicationPeerManager().removeAllQueuesAndHFileRefs(peerId);
 if (peerConfig.isSerial()) {
   env.getReplicationPeerManager().removeAllLastPushedSeqIds(peerId);

http://git-wip-us.apache.org/repos/asf/hbase/blob/94e74419/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplaySyncReplicationWALManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplaySyncReplicationWALManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplaySyncReplicationWALManager.java
index 72f5c37..eac5aa4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplaySyncReplicationWALManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplaySyncReplicationWALManager.java
@@ -115,6 +115,14 @@ public class ReplaySyncReplicationWALManager {
 }
   }
 
+  public void removePeerRemoteWALs(String peerId) throws IOException {
+Path remoteWALDir = getPeerRemoteWALDir(peerId);
+if (fs.exists(remoteWALDir) && !fs.delete(remoteWALDir, true)) {
+  throw new IOException(
+  "Failed to remove remote WALs dir " + remoteWALDir + " for peer id=" 
+ peerId);
+}
+  }
+
   public void initPeerWorkers(String peerId) {
 BlockingQueue servers = new LinkedBlockingQueue<>();
 services.getServerManager().getOnlineServers().keySet()

http://git-wip-us.apache.org/repos/asf/hbase/blob/94e74419/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java
index 0d5fce8..de679be 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.replication;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -71,6 +72,10 @@ public class SyncReplicationTestBase {
 
   protected static String PEER_ID = "1";
 
+  protected static Path remoteWALDir1;
+
+  protected static Path remoteWALDir2;
+
   p

[19/34] hbase git commit: HBASE-19864 Use protobuf instead of enum.ordinal to store SyncReplicationState

[zhangduo]

HBASE-19864 Use protobuf instead of enum.ordinal to store SyncReplicationState

Signed-off-by: zhangduo 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a81f284
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a81f284
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a81f284

Branch: refs/heads/HBASE-19064
Commit: 9a81f284b2181dc97519598997006fda59363697
Parents: b10314b
Author: Guanghao Zhang 
Authored: Fri Jan 26 16:50:48 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../replication/ReplicationPeerConfigUtil.java  | 22 +++---
 .../hbase/replication/SyncReplicationState.java | 17 ++
 .../hbase/shaded/protobuf/RequestConverter.java |  7 +++---
 .../src/main/protobuf/Replication.proto | 13 +++
 .../replication/ZKReplicationPeerStorage.java   | 24 +---
 .../hadoop/hbase/master/MasterRpcServices.java  |  9 
 ...ransitPeerSyncReplicationStateProcedure.java |  9 
 .../TestReplicationSourceManager.java   |  2 +-
 8 files changed, 67 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a81f284/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 6cbe05b..331795c 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -403,7 +403,7 @@ public final class ReplicationPeerConfigUtil {
 ReplicationProtos.ReplicationState.State.ENABLED == 
desc.getState().getState();
 ReplicationPeerConfig config = convert(desc.getConfig());
 return new ReplicationPeerDescription(desc.getId(), enabled, config,
-
SyncReplicationState.valueOf(desc.getSyncReplicationState().getNumber()));
+  toSyncReplicationState(desc.getSyncReplicationState()));
   }
 
   public static ReplicationProtos.ReplicationPeerDescription
@@ -411,17 +411,33 @@ public final class ReplicationPeerConfigUtil {
 ReplicationProtos.ReplicationPeerDescription.Builder builder =
 ReplicationProtos.ReplicationPeerDescription.newBuilder();
 builder.setId(desc.getPeerId());
+
 ReplicationProtos.ReplicationState.Builder stateBuilder =
 ReplicationProtos.ReplicationState.newBuilder();
 stateBuilder.setState(desc.isEnabled() ? 
ReplicationProtos.ReplicationState.State.ENABLED :
 ReplicationProtos.ReplicationState.State.DISABLED);
 builder.setState(stateBuilder.build());
+
 builder.setConfig(convert(desc.getPeerConfig()));
-builder.setSyncReplicationState(
-  
ReplicationProtos.SyncReplicationState.forNumber(desc.getSyncReplicationState().ordinal()));
+
builder.setSyncReplicationState(toSyncReplicationState(desc.getSyncReplicationState()));
+
 return builder.build();
   }
 
+  public static ReplicationProtos.SyncReplicationState
+  toSyncReplicationState(SyncReplicationState state) {
+ReplicationProtos.SyncReplicationState.Builder syncReplicationStateBuilder 
=
+ReplicationProtos.SyncReplicationState.newBuilder();
+syncReplicationStateBuilder
+
.setState(ReplicationProtos.SyncReplicationState.State.forNumber(state.ordinal()));
+return syncReplicationStateBuilder.build();
+  }
+
+  public static SyncReplicationState
+  toSyncReplicationState(ReplicationProtos.SyncReplicationState state) {
+return SyncReplicationState.valueOf(state.getState().getNumber());
+  }
+
   public static ReplicationPeerConfig appendTableCFsToReplicationPeerConfig(
   Map> tableCfs, ReplicationPeerConfig peerConfig) 
{
 ReplicationPeerConfigBuilder builder = 
ReplicationPeerConfig.newBuilder(peerConfig);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9a81f284/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
index bd144e9..a65b144 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
@@ -17,8 +17,15 @@
  */
 package org.apache.hadoop.hbase.replication;
 

[29/34] hbase git commit: HBASE-20370 Also remove the wal file in remote cluster when we finish replicating a file

[zhangduo]

HBASE-20370 Also remove the wal file in remote cluster when we finish 
replicating a file


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1727e6d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1727e6d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1727e6d

Branch: refs/heads/HBASE-19064
Commit: d1727e6d7c8251b4d0d97e685989fdb2c6ba8ba3
Parents: 0588a76
Author: zhangduo 
Authored: Tue Apr 17 09:04:56 2018 +0800
Committer: zhangduo 
Committed: Sat May 12 20:22:07 2018 +0800

--
 .../hbase/replication/ReplicationUtils.java |  36 ++-
 .../regionserver/ReplicationSource.java |  38 +++
 .../ReplicationSourceInterface.java |  21 +++-
 .../regionserver/ReplicationSourceManager.java  | 108 ++-
 .../regionserver/ReplicationSourceShipper.java  |  27 ++---
 .../hbase/wal/SyncReplicationWALProvider.java   |  11 +-
 .../replication/ReplicationSourceDummy.java |  20 ++--
 .../TestReplicationSourceManager.java   | 101 -
 8 files changed, 246 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d1727e6d/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index cb22f57..66e9b01 100644
--- 
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ 
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -22,14 +22,17 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CompoundConfiguration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Helper class for replication.
@@ -37,6 +40,8 @@ import org.apache.yetus.audience.InterfaceAudience;
 @InterfaceAudience.Private
 public final class ReplicationUtils {
 
+  private static final Logger LOG = 
LoggerFactory.getLogger(ReplicationUtils.class);
+
   public static final String REPLICATION_ATTR_NAME = "__rep__";
 
   public static final String REMOTE_WAL_DIR_NAME = "remoteWALs";
@@ -176,4 +181,33 @@ public final class ReplicationUtils {
   return tableCFs != null && tableCFs.containsKey(tableName);
 }
   }
+
+  public static FileSystem getRemoteWALFileSystem(Configuration conf, String 
remoteWALDir)
+  throws IOException {
+return new Path(remoteWALDir).getFileSystem(conf);
+  }
+
+  public static Path getRemoteWALDirForPeer(String remoteWALDir, String 
peerId) {
+return new Path(remoteWALDir, peerId);
+  }
+
+  /**
+   * Do the sleeping logic
+   * @param msg Why we sleep
+   * @param sleepForRetries the base sleep time.
+   * @param sleepMultiplier by how many times the default sleeping time is 
augmented
+   * @param maxRetriesMultiplier the max retry multiplier
+   * @return True if sleepMultiplier is < 
maxRetriesMultiplier
+   */
+  public static boolean sleepForRetries(String msg, long sleepForRetries, int 
sleepMultiplier,
+  int maxRetriesMultiplier) {
+try {
+  LOG.trace("{}, sleeping {} times {}", msg, sleepForRetries, 
sleepMultiplier);
+  Thread.sleep(sleepForRetries * sleepMultiplier);
+} catch (InterruptedException e) {
+  LOG.debug("Interrupted while sleeping between retries");
+  Thread.currentThread().interrupt();
+}
+return sleepMultiplier < maxRetriesMultiplier;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d1727e6d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index b05a673..01ccb11 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -89,8 +89,6 @@ public class ReplicationSource implements 
ReplicationSourceInterface {
 
   protec

[04/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.html
index 03a0b2a..cabb570 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.html
@@ -561,209 +561,206 @@
 553
 554/**
 555 * Set all fields together.
-556 * @param batch
-557 * @param sizeScope
-558 * @param dataSize
-559 */
-560void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
-561LimitScope timeScope, long time) 
{
-562  setBatch(batch);
-563  setSizeScope(sizeScope);
-564  setDataSize(dataSize);
-565  setHeapSize(heapSize);
-566  setTimeScope(timeScope);
-567  setTime(time);
-568}
-569
-570int getBatch() {
-571  return this.batch;
-572}
-573
-574void setBatch(int batch) {
-575  this.batch = batch;
-576}
-577
-578/**
-579 * @param checkerScope
-580 * @return true when the limit can be 
enforced from the scope of the checker
-581 */
-582boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
-583  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
-584}
-585
-586long getDataSize() {
-587  return this.dataSize;
-588}
-589
-590long getHeapSize() {
-591  return this.heapSize;
-592}
-593
-594void setDataSize(long dataSize) {
-595  this.dataSize = dataSize;
-596}
-597
-598void setHeapSize(long heapSize) {
-599  this.heapSize = heapSize;
-600}
-601
-602/**
-603 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
-604 */
-605LimitScope getSizeScope() {
-606  return this.sizeScope;
-607}
-608
-609/**
-610 * Change the scope in which the size 
limit is enforced
-611 */
-612void setSizeScope(LimitScope scope) 
{
-613  this.sizeScope = scope;
-614}
-615
-616/**
-617 * @param checkerScope
-618 * @return true when the limit can be 
enforced from the scope of the checker
-619 */
-620boolean 
canEnforceSizeLimitFromScope(LimitScope checkerScope) {
-621  return 
this.sizeScope.canEnforceLimitFromScope(checkerScope);
-622}
-623
-624long getTime() {
-625  return this.time;
-626}
-627
-628void setTime(long time) {
-629  this.time = time;
-630}
-631
-632/**
-633 * @return {@link LimitScope} 
indicating scope in which the time limit is enforced
-634 */
-635LimitScope getTimeScope() {
-636  return this.timeScope;
-637}
-638
-639/**
-640 * Change the scope in which the time 
limit is enforced
-641 */
-642void setTimeScope(LimitScope scope) 
{
-643  this.timeScope = scope;
-644}
-645
-646/**
-647 * @param checkerScope
-648 * @return true when the limit can be 
enforced from the scope of the checker
-649 */
-650boolean 
canEnforceTimeLimitFromScope(LimitScope checkerScope) {
-651  return 
this.timeScope.canEnforceLimitFromScope(checkerScope);
-652}
-653
-654@Override
-655public String toString() {
-656  StringBuilder sb = new 
StringBuilder();
-657  sb.append("{");
+556 */
+557void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
+558LimitScope timeScope, long time) 
{
+559  setBatch(batch);
+560  setSizeScope(sizeScope);
+561  setDataSize(dataSize);
+562  setHeapSize(heapSize);
+563  setTimeScope(timeScope);
+564  setTime(time);
+565}
+566
+567int getBatch() {
+568  return this.batch;
+569}
+570
+571void setBatch(int batch) {
+572  this.batch = batch;
+573}
+574
+575/**
+576 * @param checkerScope
+577 * @return true when the limit can be 
enforced from the scope of the checker
+578 */
+579boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
+580  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
+581}
+582
+583long getDataSize() {
+584  return this.dataSize;
+585}
+586
+587long getHeapSize() {
+588  return this.heapSize;
+589}
+590
+591void setDataSize(long dataSize) {
+592  this.dataSize = dataSize;
+593}
+594
+595void setHeapSize(long heapSize) {
+596  this.heapSize = heapSize;
+597}
+598
+599/**
+600 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
+601 */
+602LimitScope getSizeScope() {
+603  return this.sizeScope;
+604}
+605
+606/**
+607 * Change the scope in which the size 
limit is enforced
+608 */
+609void setSizeScop

[02/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/pseudo-distributed.html
--
diff --git a/pseudo-distributed.html b/pseudo-distributed.html
index 80ecf34..250c185 100644
--- a/pseudo-distributed.html
+++ b/pseudo-distributed.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase –  
 Running Apache HBase (TM) in pseudo-distributed mode
@@ -308,7 +308,7 @@ under the License. -->
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/replication.html
--
diff --git a/replication.html b/replication.html
index 8c01fee..146dbaf 100644
--- a/replication.html
+++ b/replication.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – 
   Apache HBase (TM) Replication
@@ -303,7 +303,7 @@ under the License. -->
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/resources.html
--
diff --git a/resources.html b/resources.html
index 9a7530e..b733826 100644
--- a/resources.html
+++ b/resources.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Other Apache HBase (TM) Resources
 
@@ -331,7 +331,7 @@ under the License. -->
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/source-repository.html
--
diff --git a/source-repository.html b/source-repository.html
index da1670a..3a3f6b8 100644
--- a/source-repository.html
+++ b/source-repository.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Source Code Management
 
@@ -299,7 +299,7 @@
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/sponsors.html
--
diff --git a/sponsors.html b/sponsors.html
index 0dc4bd9..94f7738 100644
--- a/sponsors.html
+++ b/sponsors.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Apache HBase™ Sponsors
 
@@ -333,7 +333,7 @@ under the License. -->
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/supportingprojects.html
--
diff --git a/supportingprojects.html b/supportingprojects.html
index 6363a22..6d7592d 100644
--- a/supportingprojects.html
+++ b/supportingprojects.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Supporting Projects
 
@@ -520,7 +520,7 @@ under the License. -->
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/team-list.html
--
diff --git a/team-list.html b/team-list.html
index 287e5a5..3352e47 100644
--- a/team-list.html
+++ b/team-list.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Project Team
 
@@ -730,7 +730,7 @@
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/testdevapidocs/org/apac

[10/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushQueueEntry.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushQueueEntry.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushQueueEntry.html
index 4c42811..0bc3ddb 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushQueueEntry.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushQueueEntry.html
@@ -563,381 +563,390 @@
 555// If this is first time we've 
been put off, then emit a log message.
 556if (fqe.getRequeueCount() <= 
0) {
 557  // Note: We don't impose 
blockingStoreFiles constraint on meta regions
-558  LOG.warn("Region " + 
region.getRegionInfo().getEncodedName() + " has too many " +
-559"store files; delaying flush 
up to " + this.blockingWaitTime + "ms");
-560  if 
(!this.server.compactSplitThread.requestSplit(region)) {
-561try {
-562  
this.server.compactSplitThread.requestSystemCompaction(region,
-563
Thread.currentThread().getName());
-564} catch (IOException e) {
-565  e = e instanceof 
RemoteException ?
-566  
((RemoteException)e).unwrapRemoteException() : e;
-567  LOG.error("Cache flush 
failed for region " +
-568
Bytes.toStringBinary(region.getRegionInfo().getRegionName()), e);
-569}
-570  }
-571}
-572
-573// Put back on the queue.  Have 
it come back out of the queue
-574// after a delay of 
this.blockingWaitTime / 100 ms.
-575
this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
-576// Tell a lie, it's not flushed 
but it's ok
-577return true;
-578  }
-579}
-580return flushRegion(region, false, 
fqe.isForceFlushAllStores(), fqe.getTracker());
-581  }
-582
-583  /**
-584   * Flush a region.
-585   * @param region Region to flush.
-586   * @param emergencyFlush Set if we are 
being force flushed. If true the region
-587   * needs to be removed from the flush 
queue. If false, when we were called
-588   * from the main flusher run loop and 
we got the entry to flush by calling
-589   * poll on the flush queue (which 
removed it).
-590   * @param forceFlushAllStores whether 
we want to flush all store.
-591   * @return true if the region was 
successfully flushed, false otherwise. If
-592   * false, there will be accompanying 
log messages explaining why the region was
-593   * not flushed.
-594   */
-595  private boolean flushRegion(HRegion 
region, boolean emergencyFlush, boolean forceFlushAllStores,
-596  FlushLifeCycleTracker tracker) {
-597synchronized (this.regionsInQueue) 
{
-598  FlushRegionEntry fqe = 
this.regionsInQueue.remove(region);
-599  // Use the start time of the 
FlushRegionEntry if available
-600  if (fqe != null && 
emergencyFlush) {
-601// Need to remove from region 
from delay queue. When NOT an
-602// emergencyFlush, then item was 
removed via a flushQueue.poll.
-603flushQueue.remove(fqe);
-604  }
-605}
-606
-607tracker.beforeExecution();
-608lock.readLock().lock();
-609try {
-610  notifyFlushRequest(region, 
emergencyFlush);
-611  FlushResult flushResult = 
region.flushcache(forceFlushAllStores, false, tracker);
-612  boolean shouldCompact = 
flushResult.isCompactionNeeded();
-613  // We just want to check the size
-614  boolean shouldSplit = 
region.checkSplit() != null;
-615  if (shouldSplit) {
-616
this.server.compactSplitThread.requestSplit(region);
-617  } else if (shouldCompact) {
-618
server.compactSplitThread.requestSystemCompaction(region, 
Thread.currentThread().getName());
-619  }
-620} catch (DroppedSnapshotException ex) 
{
-621  // Cache flush can fail in a few 
places. If it fails in a critical
-622  // section, we get a 
DroppedSnapshotException and a replay of wal
-623  // is required. Currently the only 
way to do this is a restart of
-624  // the server. Abort because hdfs 
is probably bad (HBASE-644 is a case
-625  // where hdfs was bad but passed 
the hdfs check).
-626  server.abort("Replay of WAL 
required. Forcing server shutdown", ex);
-627  return false;
-628} catch (IOException ex) {
-629  ex = ex instanceof RemoteException 
? ((RemoteException) ex).unwrapRemoteException() : ex;
-630  LOG.error(
-631"Cache flush failed"
-632+ (region != null ? (" for 
region " +
-633
Bytes.toStringBinary(region.getRegionInfo().getRegionName()))
-634  : ""), ex);
-635  if (!server.checkFileSystem()) {
-636return false;
-637  }
-

[18/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public R

[03/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/Segment.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/Segment.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/Segment.html
index 4d2c914..0b1cae9 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/Segment.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/Segment.html
@@ -53,7 +53,7 @@
 045 * segments from active set to snapshot 
set in the default implementation.
 046 */
 047@InterfaceAudience.Private
-048public abstract class Segment {
+048public abstract class Segment implements 
MemStoreSizing {
 049
 050  public final static long FIXED_OVERHEAD 
= ClassSize.align(ClassSize.OBJECT
 051  + 5 * ClassSize.REFERENCE // 
cellSet, comparator, memStoreLAB, memStoreSizing,
@@ -67,9 +67,9 @@
 059  private final CellComparator 
comparator;
 060  protected long minSequenceId;
 061  private MemStoreLAB memStoreLAB;
-062  // Sum of sizes of all Cells added to 
this Segment. Cell's heapSize is considered. This is not
+062  // Sum of sizes of all Cells added to 
this Segment. Cell's HeapSize is considered. This is not
 063  // including the heap overhead of this 
class.
-064  protected final MemStoreSizing 
segmentSize;
+064  protected final MemStoreSizing 
memStoreSizing;
 065  protected final TimeRangeTracker 
timeRangeTracker;
 066  protected volatile boolean 
tagsPresent;
 067
@@ -77,352 +77,348 @@
 069  // and there is no need in true 
Segments state
 070  protected Segment(CellComparator 
comparator, TimeRangeTracker trt) {
 071this.comparator = comparator;
-072this.segmentSize = new 
MemStoreSizing();
-073this.timeRangeTracker = trt;
-074  }
-075
-076  protected Segment(CellComparator 
comparator, List segments,
-077  TimeRangeTracker trt) {
-078long dataSize = 0;
-079long heapSize = 0;
-080long OffHeapSize = 0;
-081for (Segment segment : segments) {
-082  MemStoreSize memStoreSize = 
segment.getMemStoreSize();
-083  dataSize += 
memStoreSize.getDataSize();
-084  heapSize += 
memStoreSize.getHeapSize();
-085  OffHeapSize += 
memStoreSize.getOffHeapSize();
-086}
-087this.comparator = comparator;
-088this.segmentSize = new 
MemStoreSizing(dataSize, heapSize, OffHeapSize);
-089this.timeRangeTracker = trt;
-090  }
-091
-092  // This constructor is used to create 
empty Segments.
-093  protected Segment(CellSet cellSet, 
CellComparator comparator, MemStoreLAB memStoreLAB, TimeRangeTracker trt) {
-094this.cellSet.set(cellSet);
-095this.comparator = comparator;
-096this.minSequenceId = 
Long.MAX_VALUE;
-097this.memStoreLAB = memStoreLAB;
-098this.segmentSize = new 
MemStoreSizing();
-099this.tagsPresent = false;
-100this.timeRangeTracker = trt;
-101  }
-102
-103  protected Segment(Segment segment) {
-104
this.cellSet.set(segment.getCellSet());
-105this.comparator = 
segment.getComparator();
-106this.minSequenceId = 
segment.getMinSequenceId();
-107this.memStoreLAB = 
segment.getMemStoreLAB();
-108this.segmentSize = new 
MemStoreSizing(segment.getMemStoreSize());
-109this.tagsPresent = 
segment.isTagsPresent();
-110this.timeRangeTracker = 
segment.getTimeRangeTracker();
-111  }
-112
-113  /**
-114   * Creates the scanner for the given 
read point
-115   * @return a scanner for the given read 
point
-116   */
-117  protected KeyValueScanner 
getScanner(long readPoint) {
-118return new SegmentScanner(this, 
readPoint);
-119  }
-120
-121  public List 
getScanners(long readPoint) {
-122return Collections.singletonList(new 
SegmentScanner(this, readPoint));
-123  }
-124
-125  /**
-126   * @return whether the segment has any 
cells
-127   */
-128  public boolean isEmpty() {
-129return getCellSet().isEmpty();
-130  }
-131
-132  /**
-133   * @return number of cells in segment
-134   */
-135  public int getCellsCount() {
-136return getCellSet().size();
-137  }
-138
-139  /**
-140   * Closing a segment before it is being 
discarded
-141   */
-142  public void close() {
-143if (this.memStoreLAB != null) {
-144  this.memStoreLAB.close();
-145}
-146// do not set MSLab to null as 
scanners may still be reading the data here and need to decrease
-147// the counter when they finish
-148  }
-149
-150  /**
-151   * If the segment has a memory 
allocator the cell is being cloned to this space, and returned;
-152   * otherwise the given cell is 
returned
-153   *
-154   * When a cell's size is too big 
(bigger than maxAlloc), it is not allocated on MSLAB.
-155   * Since the process of flattening to 
CellChunkMap assumes that all cells
-156   * are allocated on MSLAB, during this 
process, the input parameter
-157   * forceCloneOfBigCell is set to 'true' 
and the cell is copied

hbase-site git commit: INFRA-10751 Empty commit

[git-site-role]

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 92a26cfbe -> e3e9a0110


INFRA-10751 Empty commit


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/e3e9a011
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/e3e9a011
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/e3e9a011

Branch: refs/heads/asf-site
Commit: e3e9a01105c6193ddcb439457550d4a7dde79db1
Parents: 92a26cf
Author: jenkins 
Authored: Sat May 12 14:47:48 2018 +
Committer: jenkins 
Committed: Sat May 12 14:47:48 2018 +

--

--

[22/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResultImpl.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResultImpl.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResultImpl.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResultImpl.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResultImpl.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServi

[51/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/92a26cfb
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/92a26cfb
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/92a26cfb

Branch: refs/heads/asf-site
Commit: 92a26cfbe8d2f36231320ca41677e6757d891e81
Parents: a8dd4f3
Author: jenkins 
Authored: Sat May 12 14:47:27 2018 +
Committer: jenkins 
Committed: Sat May 12 14:47:27 2018 +

--
 acid-semantics.html | 4 +-
 apache_hbase_reference_guide.pdf| 4 +-
 book.html   | 2 +-
 bulk-loads.html | 4 +-
 checkstyle-aggregate.html   | 12722 +++---
 checkstyle.rss  |36 +-
 coc.html| 4 +-
 dependencies.html   | 4 +-
 dependency-convergence.html | 4 +-
 dependency-info.html| 4 +-
 dependency-management.html  | 4 +-
 devapidocs/allclasses-frame.html| 4 +-
 devapidocs/allclasses-noframe.html  | 4 +-
 devapidocs/constant-values.html | 6 +-
 devapidocs/index-all.html   |   165 +-
 .../hadoop/hbase/backup/package-tree.html   | 4 +-
 .../org/apache/hadoop/hbase/class-use/Cell.html |44 +-
 .../hadoop/hbase/client/package-tree.html   |26 +-
 .../hadoop/hbase/executor/package-tree.html | 2 +-
 .../hadoop/hbase/filter/package-tree.html   | 6 +-
 .../hadoop/hbase/io/hfile/package-tree.html | 6 +-
 .../apache/hadoop/hbase/ipc/package-tree.html   | 2 +-
 .../hadoop/hbase/mapreduce/package-tree.html| 4 +-
 .../hadoop/hbase/master/package-tree.html   | 4 +-
 .../hbase/master/procedure/package-tree.html| 4 +-
 .../hadoop/hbase/monitoring/package-tree.html   | 2 +-
 .../org/apache/hadoop/hbase/package-tree.html   |20 +-
 .../hadoop/hbase/procedure2/package-tree.html   | 4 +-
 .../hadoop/hbase/quotas/package-tree.html   | 8 +-
 .../hbase/regionserver/AbstractMemStore.html|87 +-
 .../regionserver/CSLMImmutableSegment.html  |22 +-
 .../regionserver/CellArrayImmutableSegment.html |26 +-
 .../regionserver/CellChunkImmutableSegment.html |26 +-
 ...ompactingMemStore.InMemoryFlushRunnable.html | 6 +-
 .../hbase/regionserver/CompactingMemStore.html  |82 +-
 .../hbase/regionserver/CompactionPipeline.html  |72 +-
 .../regionserver/CompositeImmutableSegment.html |   116 +-
 .../hbase/regionserver/DefaultMemStore.html |36 +-
 .../HRegion.BatchOperation.Visitor.html | 4 +-
 .../regionserver/HRegion.BatchOperation.html|84 +-
 .../regionserver/HRegion.BulkLoadListener.html  | 8 +-
 .../HRegion.FlushResult.Result.html |10 +-
 .../hbase/regionserver/HRegion.FlushResult.html | 8 +-
 .../HRegion.MutationBatchOperation.html |44 +-
 .../HRegion.PrepareFlushResult.html |12 +-
 .../regionserver/HRegion.RegionScannerImpl.html |90 +-
 .../HRegion.ReplayBatchOperation.html   |32 +-
 .../regionserver/HRegion.RowLockContext.html|28 +-
 .../hbase/regionserver/HRegion.RowLockImpl.html |16 +-
 .../hadoop/hbase/regionserver/HRegion.html  |  1008 +-
 .../hadoop/hbase/regionserver/HStore.html   |12 +-
 .../hbase/regionserver/ImmutableSegment.html|22 +-
 .../hadoop/hbase/regionserver/MemStore.html |44 +-
 .../MemStoreFlusher.FlushQueueEntry.html| 2 +-
 .../MemStoreFlusher.FlushRegionEntry.html   |36 +-
 .../hbase/regionserver/MemStoreFlusher.html |79 +-
 .../hadoop/hbase/regionserver/MemStoreSize.html |82 +-
 .../hbase/regionserver/MemStoreSizing.html  |   234 +-
 .../hbase/regionserver/MemStoreSnapshot.html| 4 +-
 .../hbase/regionserver/MutableSegment.html  |30 +-
 .../NoSuchColumnFamilyException.html| 4 +-
 .../NonReversedNonLazyKeyValueScanner.html  | 4 +-
 .../NonThreadSafeMemStoreSizing.html|   489 +
 .../regionserver/RegionServerAccounting.html|52 +-
 .../regionserver/RegionServicesForStores.html   |10 +-
 .../ScannerContext.LimitFields.html |40 +-
 .../ScannerContext.ProgressFields.html  |32 +-
 .../hadoop/hbase/regionserver/Segment.html  |   249 +-
 .../hbase/regionserver/SegmentFactory.html  | 4 +-
 .../StripeStoreFlusher.StripeFlushRequest.html  | 4 +-
 .../regionserver/ThreadSafeMemStoreSizing.html  |   487 +
 .../hbase/regionserver/TimeRangeTracker.html| 4 +-
 .../class-use/CSLMImmutableSegment.html  

[35/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.

[git-site-role]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/MemStoreSizing.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/MemStoreSizing.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/MemStoreSizing.html
index 47c0f34..075c23d 100644
--- 
a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/MemStoreSizing.html
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/MemStoreSizing.html
@@ -4,7 +4,7 @@
 
 
 
-Uses of Class org.apache.hadoop.hbase.regionserver.MemStoreSizing 
(Apache HBase 3.0.0-SNAPSHOT API)
+Uses of Interface org.apache.hadoop.hbase.regionserver.MemStoreSizing 
(Apache HBase 3.0.0-SNAPSHOT API)
 
 
 
@@ -12,7 +12,7 @@
 
 

-
Uses of 
Class
org.apache.hadoop.hbase.regionserver.MemStoreSizing
+
Uses of 
Interface
org.apache.hadoop.hbase.regionserver.MemStoreSizing
 
 

 

 

 
-
+
@@ -94,68 +94,116 @@
 

 
 
-
Uses of MemStoreSizing in org.apache.hadoop.hbase.regionserver
-
Packages that use MemStoreSizing 
Packages that use MemStoreSizing 
 

 
Package
 
Description
-
+
Uses of MemStoreSizing 
in org.apache.hadoop.hbase.regionserver
+
Fields in org.apache.hadoop.hbase.regionserver
 declared as MemStoreSizing 
+
Classes in org.apache.hadoop.hbase.regionserver
 that implement MemStoreSizing 
 

 
M



[40/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HStore.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/HStore.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HStore.html
index 3735b20..905969d 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/HStore.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/HStore.html
@@ -425,14 +425,14 @@ implements 
 void
 add(Cell cell,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Adds a value to the memstore
 
 
 
 void
 add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Adds the specified value to the memstore
 
 
@@ -1216,7 +1216,7 @@ implements void
 upsert(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
   long readpoint,
-  MemStoreSizing memstoreSizing)
+  MemStoreSizing memstoreSizing)
 Adds or replaces the specified KeyValues.
 
 
@@ -2305,7 +2305,7 @@ public static org.apache.hadoop.fs.Path 
 add
 public void add(Cell cell,
-MemStoreSizing memstoreSizing)
+MemStoreSizing memstoreSizing)
 Adds a value to the memstore
 
 
@@ -2316,7 +2316,7 @@ public static org.apache.hadoop.fs.Path 
 add
 public void add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-MemStoreSizing memstoreSizing)
+MemStoreSizing memstoreSizing)
 Adds the specified value to the memstore
 
 
@@ -3664,7 +3664,7 @@ public static org.apache.hadoop.fs.Path upsert(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
long readpoint,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 throws https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true";
 title="class or interface in java.io">IOException
 Adds or replaces the specified KeyValues.
  

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/ImmutableSegment.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/ImmutableSegment.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/ImmutableSegment.html
index b7b243d..f2cf000 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/ImmutableSegment.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/ImmutableSegment.html
@@ -113,6 +113,10 @@ var activeTableTab = "activeTableTab";
 
 
 
+All Implemented Interfaces:
+MemStoreSizing
+
+
 Direct Known Subclasses:
 CellArrayImmutableSegment, CellChunkImmutableSegment, CompositeImmutableSegment, CSLMImmutableSegment
 
@@ -151,7 +155,14 @@ extends Segment
-FIXED_OVERHEAD,
 minSequenceId,
 segmentSize,
 tagsPresent,
 timeRangeTracker
+FIXED_OVERHEAD,
 memStoreSizing,
 minSequenceId,
 tagsPresent,
 timeRangeTracker
+
+
+
+
+
+Fields inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+DUD
 
 
 
@@ -236,7 +247,7 @@ extends Segment
-close,
 compare,
 compareRows,
 decScannerCount,
 dump,
 getCellLength,
 getCellsCount,
 getCellSet,
 getComparator,
 getMemStoreLAB,
 getMemStoreSize,
 getMinSequenceId,
 getScanner,
 getScanners,
 getTimeRangeTracker,
 headSet,
 heapSize, heapSizeChange,
 incScannerCount,
 incSize,
 indexEntryOffHeapSize,
 indexEntryOnHeapSize,
 indexEntrySize,
 internalAdd,
 isEmpty, isTagsPresent,
 iterator,
 keySize,
 last,
 maybeCloneWithAllocator,
 offHeapSize,
 offHeapSizeChange,
 setCellSet,
 shouldSeek,
 tailSet,
 updateMetaInfo,
 updateMetaInfo
+close,
 compare,
 compareRows,
 decScannerCount,
 dump,
 getCellLength,
 getCellsCount,
 getCellSet,
 getComparator,
 getDataSize,
 getHeapSize,
 getMemStoreLAB,
 getMemStoreSize,
 getMinSequenceId,
 getOffHeapSize,
 getScanner,
 getScanners,
 getTimeRangeTracker,
 headSet,
 heapSizeChange,
 incMemStoreSize,
 incScannerCount,
 indexEntryOffHeapSize,
 indexEntryOnHeapSize,
 indexEntryS
 ize, internalAdd,
 isEmpty,
 isTagsPresent,
 iterator,
 last,
 maybeCloneWithAllocator,
 offHeapSizeChange,
 setCellSet,
 shouldSeek,
 tailSet,
 updateMetaInfo,
 updateMetaInfo
 
 
 
@@ -245,6 +256,13 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
 https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#clone--";
 title="class or interface in java.lang">clone, https://docs.oracle



[25/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public Region



[08/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.html
index 4c42811..0bc3ddb 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.html
@@ -563,381 +563,390 @@
 555// If this is first time we've 
been put off, then emit a log message.
 556if (fqe.getRequeueCount() <= 
0) {
 557  // Note: We don't impose 
blockingStoreFiles constraint on meta regions
-558  LOG.warn("Region " + 
region.getRegionInfo().getEncodedName() + " has too many " +
-559"store files; delaying flush 
up to " + this.blockingWaitTime + "ms");
-560  if 
(!this.server.compactSplitThread.requestSplit(region)) {
-561try {
-562  
this.server.compactSplitThread.requestSystemCompaction(region,
-563
Thread.currentThread().getName());
-564} catch (IOException e) {
-565  e = e instanceof 
RemoteException ?
-566  
((RemoteException)e).unwrapRemoteException() : e;
-567  LOG.error("Cache flush 
failed for region " +
-568
Bytes.toStringBinary(region.getRegionInfo().getRegionName()), e);
-569}
-570  }
-571}
-572
-573// Put back on the queue.  Have 
it come back out of the queue
-574// after a delay of 
this.blockingWaitTime / 100 ms.
-575
this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
-576// Tell a lie, it's not flushed 
but it's ok
-577return true;
-578  }
-579}
-580return flushRegion(region, false, 
fqe.isForceFlushAllStores(), fqe.getTracker());
-581  }
-582
-583  /**
-584   * Flush a region.
-585   * @param region Region to flush.
-586   * @param emergencyFlush Set if we are 
being force flushed. If true the region
-587   * needs to be removed from the flush 
queue. If false, when we were called
-588   * from the main flusher run loop and 
we got the entry to flush by calling
-589   * poll on the flush queue (which 
removed it).
-590   * @param forceFlushAllStores whether 
we want to flush all store.
-591   * @return true if the region was 
successfully flushed, false otherwise. If
-592   * false, there will be accompanying 
log messages explaining why the region was
-593   * not flushed.
-594   */
-595  private boolean flushRegion(HRegion 
region, boolean emergencyFlush, boolean forceFlushAllStores,
-596  FlushLifeCycleTracker tracker) {
-597synchronized (this.regionsInQueue) 
{
-598  FlushRegionEntry fqe = 
this.regionsInQueue.remove(region);
-599  // Use the start time of the 
FlushRegionEntry if available
-600  if (fqe != null && 
emergencyFlush) {
-601// Need to remove from region 
from delay queue. When NOT an
-602// emergencyFlush, then item was 
removed via a flushQueue.poll.
-603flushQueue.remove(fqe);
-604  }
-605}
-606
-607tracker.beforeExecution();
-608lock.readLock().lock();
-609try {
-610  notifyFlushRequest(region, 
emergencyFlush);
-611  FlushResult flushResult = 
region.flushcache(forceFlushAllStores, false, tracker);
-612  boolean shouldCompact = 
flushResult.isCompactionNeeded();
-613  // We just want to check the size
-614  boolean shouldSplit = 
region.checkSplit() != null;
-615  if (shouldSplit) {
-616
this.server.compactSplitThread.requestSplit(region);
-617  } else if (shouldCompact) {
-618
server.compactSplitThread.requestSystemCompaction(region, 
Thread.currentThread().getName());
-619  }
-620} catch (DroppedSnapshotException ex) 
{
-621  // Cache flush can fail in a few 
places. If it fails in a critical
-622  // section, we get a 
DroppedSnapshotException and a replay of wal
-623  // is required. Currently the only 
way to do this is a restart of
-624  // the server. Abort because hdfs 
is probably bad (HBASE-644 is a case
-625  // where hdfs was bad but passed 
the hdfs check).
-626  server.abort("Replay of WAL 
required. Forcing server shutdown", ex);
-627  return false;
-628} catch (IOException ex) {
-629  ex = ex instanceof RemoteException 
? ((RemoteException) ex).unwrapRemoteException() : ex;
-630  LOG.error(
-631"Cache flush failed"
-632+ (region != null ? (" for 
region " +
-633
Bytes.toStringBinary(region.getRegionInfo().getRegionName()))
-634  : ""), ex);
-635  if (!server.checkFileSystem()) {
-636return false;
-637  }
-638} finally {
-639  lock.readLock().unlock();
-640  wakeUpIfBlocking



[30/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
index d4390be..1037b84 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
@@ -163,12 +163,12 @@
 155   */
 156  @Override
 157  public MemStoreSize size() {
-158MemStoreSizing memstoreSizing = new 
MemStoreSizing();
+158MemStoreSizing memstoreSizing = new 
NonThreadSafeMemStoreSizing();
 159
memstoreSizing.incMemStoreSize(active.getMemStoreSize());
 160for (Segment item : 
pipeline.getSegments()) {
 161  
memstoreSizing.incMemStoreSize(item.getMemStoreSize());
 162}
-163return memstoreSizing;
+163return 
memstoreSizing.getMemStoreSize();
 164  }
 165
 166  /**
@@ -224,369 +224,365 @@
 216return new 
MemStoreSnapshot(snapshotId, this.snapshot);
 217  }
 218
-219  /**
-220   * On flush, how much memory we will 
clear.
-221   * @return size of data that is going 
to be flushed
-222   */
-223  @Override
-224  public MemStoreSize getFlushableSize() 
{
-225MemStoreSizing snapshotSizing = 
getSnapshotSizing();
-226if (snapshotSizing.getDataSize() == 
0) {
-227  // if snapshot is empty the tail of 
the pipeline (or everything in the memstore) is flushed
-228  if (compositeSnapshot) {
-229snapshotSizing = 
pipeline.getPipelineSizing();
-230
snapshotSizing.incMemStoreSize(active.getMemStoreSize());
-231  } else {
-232snapshotSizing = 
pipeline.getTailSizing();
-233  }
-234}
-235return snapshotSizing.getDataSize() 
> 0 ? snapshotSizing
-236: new 
MemStoreSize(active.getMemStoreSize());
-237  }
-238
-239  @Override
-240  protected long keySize() {
-241// Need to consider keySize of all 
segments in pipeline and active
-242long k = this.active.keySize();
-243for (Segment segment : 
this.pipeline.getSegments()) {
-244  k += segment.keySize();
-245}
-246return k;
-247  }
-248
-249  @Override
-250  protected long heapSize() {
-251// Need to consider heapOverhead of 
all segments in pipeline and active
-252long h = this.active.heapSize();
-253for (Segment segment : 
this.pipeline.getSegments()) {
-254  h += segment.heapSize();
-255}
-256return h;
-257  }
-258
-259  @Override
-260  public void 
updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater) {
-261long minSequenceId = 
pipeline.getMinSequenceId();
-262if(minSequenceId != Long.MAX_VALUE) 
{
-263  byte[] encodedRegionName = 
getRegionServices().getRegionInfo().getEncodedNameAsBytes();
-264  byte[] familyName = 
getFamilyNameInBytes();
-265  WAL WAL = 
getRegionServices().getWAL();
-266  if (WAL != null) {
-267
WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater);
-268  }
-269}
-270  }
-271
-272  /**
-273   * This message intends to inform the 
MemStore that next coming updates
-274   * are going to be part of the 
replaying edits from WAL
-275   */
-276  @Override
-277  public void startReplayingFromWAL() {
-278inWalReplay = true;
-279  }
-280
-281  /**
-282   * This message intends to inform the 
MemStore that the replaying edits from WAL
-283   * are done
-284   */
-285  @Override
-286  public void stopReplayingFromWAL() {
-287inWalReplay = false;
-288  }
-289
-290  // the getSegments() method is used for 
tests only
-291  @VisibleForTesting
-292  @Override
-293  protected List 
getSegments() {
-294List 
pipelineList = pipeline.getSegments();
-295List list = new 
ArrayList<>(pipelineList.size() + 2);
-296list.add(this.active);
-297list.addAll(pipelineList);
-298
list.addAll(this.snapshot.getAllSegments());
-299
-300return list;
-301  }
-302
-303  // the following three methods allow to 
manipulate the settings of composite snapshot
-304  public void 
setCompositeSnapshot(boolean useCompositeSnapshot) {
-305this.compositeSnapshot = 
useCompositeSnapshot;
-306  }
-307
-308  public boolean 
swapCompactedSegments(VersionedSegmentsList versionedList, ImmutableSegment 
result,
-309  boolean merge) {
-310// last true stands for updating the 
region size
-311return pipeline.swap(versionedList, 
result, !merge, true);
-312  }
-313
-314  /**
-315   * @param requesterVersion The caller 
must hold the VersionedList of the pipeline
-316   *   with version taken 
earlier. This version must be passed as a parameter here.
-317   *   The flattening happens 
only if versions match.
-318   */
-319  public void flattenOneSegment(long 
requesterVersion,  Mem



[13/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.html
index 4a879bb..7d27402 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesForStores 
getRegionServicesForStores() {
-1292return regionServicesForStore



[49/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/checkstyle.rss
--
diff --git a/checkstyle.rss b/checkstyle.rss
index f7de9ff..9cde794 100644
--- a/checkstyle.rss
+++ b/checkstyle.rss
@@ -25,8 +25,8 @@ under the License.
 en-us
 ©2007 - 2018 The Apache Software Foundation
 
-  File: 3610,
- Errors: 15878,
+  File: 3612,
+ Errors: 15873,
  Warnings: 0,
  Infos: 0
   
@@ -564,6 +564,20 @@ under the License.
   
   
 
+  http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.regionserver.NonThreadSafeMemStoreSizing.java";>org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java
+
+
+  0
+
+
+  0
+
+
+  0
+
+  
+  
+
   http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.security.access.Permission.java";>org/apache/hadoop/hbase/security/access/Permission.java
 
 
@@ -23995,7 +24009,7 @@ under the License.
   0
 
 
-  12
+  9
 
   
   
@@ -26753,7 +26767,7 @@ under the License.
   0
 
 
-  6
+  4
 
   
   
@@ -37118,6 +37132,20 @@ under the License.
   
   
 
+  http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.regionserver.ThreadSafeMemStoreSizing.java";>org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java
+
+
+  0
+
+
+  0
+
+
+  0
+
+  
+  
+
   http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.http.TestSSLHttpServer.java";>org/apache/hadoop/hbase/http/TestSSLHttpServer.java
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/coc.html
--
diff --git a/coc.html b/coc.html
index ac90537..6657441c 100644
--- a/coc.html
+++ b/coc.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – 
   Code of Conduct Policy
@@ -375,7 +375,7 @@ email to mailto:priv...@hbase.apache.org";>the priv
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/dependencies.html
--
diff --git a/dependencies.html b/dependencies.html
index d2e3e4f..00214b6 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Project Dependencies
 
@@ -440,7 +440,7 @@
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/dependency-convergence.html
--
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 85a38bd..8871f6e 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Reactor Dependency Convergence
 
@@ -1105,7 +1105,7 @@
 https://www.apache.org/";>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-05-11
+  Last Published: 
2018-05-12
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/dependency-info.html
--
diff --git a/dependency-info.html b/dependency-info.html
index 2ce1b8c..b2d4953 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Dependency Information
 
@@ -313,7 +313,7 @@
 https://www.



[48/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html 
b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
index 35b548a..52c7f71 100644
--- a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
@@ -167,10 +167,10 @@
 
 java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true";
 title="class or interface in java.lang">Enum (implements java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true";
 title="class or interface in java.lang">Comparable, java.io.https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true";
 title="class or interface in java.io">Serializable)
 
+org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase
+org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand
 org.apache.hadoop.hbase.backup.BackupType
 org.apache.hadoop.hbase.backup.BackupInfo.BackupState
-org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand
-org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/class-use/Cell.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/class-use/Cell.html 
b/devapidocs/org/apache/hadoop/hbase/class-use/Cell.html
index cf0947e..d2e954c 100644
--- a/devapidocs/org/apache/hadoop/hbase/class-use/Cell.html
+++ b/devapidocs/org/apache/hadoop/hbase/class-use/Cell.html
@@ -6002,26 +6002,26 @@ service.
 void
 MutableSegment.add(Cell cell,
boolean mslabUsed,
-   MemStoreSizing memStoreSizing)
+   MemStoreSizing memStoreSizing)
 Adds the given cell into the segment
 
 
 
 void
 AbstractMemStore.add(Cell cell,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
 
 void
 HStore.add(Cell cell,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Adds a value to the memstore
 
 
 
 void
 MemStore.add(Cell cell,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Write an update
 
 
@@ -6059,7 +6059,7 @@ service.
 private void
 HRegion.applyToMemStore(HStore store,
Cell cell,
-   MemStoreSizing memstoreAccounting) 
+   MemStoreSizing memstoreAccounting) 
 
 
 boolean
@@ -6369,19 +6369,19 @@ service.
 protected void
 CompositeImmutableSegment.internalAdd(Cell cell,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
 
 private void
 AbstractMemStore.internalAdd(Cell toAdd,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
 
 protected void
 Segment.internalAdd(Cell cell,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
 
 private boolean
@@ -6652,7 +6652,7 @@ service.
 protected void
 HRegion.restoreEdit(HStore s,
Cell cell,
-   MemStoreSizing memstoreAccounting)
+   MemStoreSizing memstoreAccounting)
 Used by tests
 
 
@@ -6911,32 +6911,32 @@ service.
 CompositeImmutableSegment.updateMetaInfo(Cell cellToAdd,
   boolean succ,
   boolean mslabUsed,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
 
 protected void
 Segment.updateMetaInfo(Cell cellToAdd,
   boolean succ,
   boolean mslabUsed,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
 
 protected void
 Segment.updateMetaInfo(Cell cellToAdd,
   boolean succ,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
 
 private void
 AbstractMemStore.upsert(Cell cell,
   long readpoint,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
 
 void
 MutableSegment.upsert(Cell cell,
   long readpoint,
-  MemStoreSizing memStoreSizing) 
+  MemStoreSizing memStoreSizing) 
 
 
 
@@ -6950,19 +6950,19 @@ service.
 
 void
 AbstractMemStore.add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
 
 void
 HStore.add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Adds the specified value to the memstore
 
 
 
 void
 MemStore.add(https://docs.oracle.com/javase/8/docs/api/java/lan



[05/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html
index 03a0b2a..cabb570 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html
@@ -561,209 +561,206 @@
 553
 554/**
 555 * Set all fields together.
-556 * @param batch
-557 * @param sizeScope
-558 * @param dataSize
-559 */
-560void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
-561LimitScope timeScope, long time) 
{
-562  setBatch(batch);
-563  setSizeScope(sizeScope);
-564  setDataSize(dataSize);
-565  setHeapSize(heapSize);
-566  setTimeScope(timeScope);
-567  setTime(time);
-568}
-569
-570int getBatch() {
-571  return this.batch;
-572}
-573
-574void setBatch(int batch) {
-575  this.batch = batch;
-576}
-577
-578/**
-579 * @param checkerScope
-580 * @return true when the limit can be 
enforced from the scope of the checker
-581 */
-582boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
-583  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
-584}
-585
-586long getDataSize() {
-587  return this.dataSize;
-588}
-589
-590long getHeapSize() {
-591  return this.heapSize;
-592}
-593
-594void setDataSize(long dataSize) {
-595  this.dataSize = dataSize;
-596}
-597
-598void setHeapSize(long heapSize) {
-599  this.heapSize = heapSize;
-600}
-601
-602/**
-603 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
-604 */
-605LimitScope getSizeScope() {
-606  return this.sizeScope;
-607}
-608
-609/**
-610 * Change the scope in which the size 
limit is enforced
-611 */
-612void setSizeScope(LimitScope scope) 
{
-613  this.sizeScope = scope;
-614}
-615
-616/**
-617 * @param checkerScope
-618 * @return true when the limit can be 
enforced from the scope of the checker
-619 */
-620boolean 
canEnforceSizeLimitFromScope(LimitScope checkerScope) {
-621  return 
this.sizeScope.canEnforceLimitFromScope(checkerScope);
-622}
-623
-624long getTime() {
-625  return this.time;
-626}
-627
-628void setTime(long time) {
-629  this.time = time;
-630}
-631
-632/**
-633 * @return {@link LimitScope} 
indicating scope in which the time limit is enforced
-634 */
-635LimitScope getTimeScope() {
-636  return this.timeScope;
-637}
-638
-639/**
-640 * Change the scope in which the time 
limit is enforced
-641 */
-642void setTimeScope(LimitScope scope) 
{
-643  this.timeScope = scope;
-644}
-645
-646/**
-647 * @param checkerScope
-648 * @return true when the limit can be 
enforced from the scope of the checker
-649 */
-650boolean 
canEnforceTimeLimitFromScope(LimitScope checkerScope) {
-651  return 
this.timeScope.canEnforceLimitFromScope(checkerScope);
-652}
-653
-654@Override
-655public String toString() {
-656  StringBuilder sb = new 
StringBuilder();
-657  sb.append("{");
+556 */
+557void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
+558LimitScope timeScope, long time) 
{
+559  setBatch(batch);
+560  setSizeScope(sizeScope);
+561  setDataSize(dataSize);
+562  setHeapSize(heapSize);
+563  setTimeScope(timeScope);
+564  setTime(time);
+565}
+566
+567int getBatch() {
+568  return this.batch;
+569}
+570
+571void setBatch(int batch) {
+572  this.batch = batch;
+573}
+574
+575/**
+576 * @param checkerScope
+577 * @return true when the limit can be 
enforced from the scope of the checker
+578 */
+579boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
+580  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
+581}
+582
+583long getDataSize() {
+584  return this.dataSize;
+585}
+586
+587long getHeapSize() {
+588  return this.heapSize;
+589}
+590
+591void setDataSize(long dataSize) {
+592  this.dataSize = dataSize;
+593}
+594
+595void setHeapSize(long heapSize) {
+596  this.heapSize = heapSize;
+597}
+598
+599/**
+600 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
+601 */
+602LimitScope getSizeScope() {
+603  return this.sizeScope;
+604}
+605
+606/**
+607 * Change the scope in which the size 
lim



[09/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushRegionEntry.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushRegionEntry.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushRegionEntry.html
index 4c42811..0bc3ddb 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushRegionEntry.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushRegionEntry.html
@@ -563,381 +563,390 @@
 555// If this is first time we've 
been put off, then emit a log message.
 556if (fqe.getRequeueCount() <= 
0) {
 557  // Note: We don't impose 
blockingStoreFiles constraint on meta regions
-558  LOG.warn("Region " + 
region.getRegionInfo().getEncodedName() + " has too many " +
-559"store files; delaying flush 
up to " + this.blockingWaitTime + "ms");
-560  if 
(!this.server.compactSplitThread.requestSplit(region)) {
-561try {
-562  
this.server.compactSplitThread.requestSystemCompaction(region,
-563
Thread.currentThread().getName());
-564} catch (IOException e) {
-565  e = e instanceof 
RemoteException ?
-566  
((RemoteException)e).unwrapRemoteException() : e;
-567  LOG.error("Cache flush 
failed for region " +
-568
Bytes.toStringBinary(region.getRegionInfo().getRegionName()), e);
-569}
-570  }
-571}
-572
-573// Put back on the queue.  Have 
it come back out of the queue
-574// after a delay of 
this.blockingWaitTime / 100 ms.
-575
this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
-576// Tell a lie, it's not flushed 
but it's ok
-577return true;
-578  }
-579}
-580return flushRegion(region, false, 
fqe.isForceFlushAllStores(), fqe.getTracker());
-581  }
-582
-583  /**
-584   * Flush a region.
-585   * @param region Region to flush.
-586   * @param emergencyFlush Set if we are 
being force flushed. If true the region
-587   * needs to be removed from the flush 
queue. If false, when we were called
-588   * from the main flusher run loop and 
we got the entry to flush by calling
-589   * poll on the flush queue (which 
removed it).
-590   * @param forceFlushAllStores whether 
we want to flush all store.
-591   * @return true if the region was 
successfully flushed, false otherwise. If
-592   * false, there will be accompanying 
log messages explaining why the region was
-593   * not flushed.
-594   */
-595  private boolean flushRegion(HRegion 
region, boolean emergencyFlush, boolean forceFlushAllStores,
-596  FlushLifeCycleTracker tracker) {
-597synchronized (this.regionsInQueue) 
{
-598  FlushRegionEntry fqe = 
this.regionsInQueue.remove(region);
-599  // Use the start time of the 
FlushRegionEntry if available
-600  if (fqe != null && 
emergencyFlush) {
-601// Need to remove from region 
from delay queue. When NOT an
-602// emergencyFlush, then item was 
removed via a flushQueue.poll.
-603flushQueue.remove(fqe);
-604  }
-605}
-606
-607tracker.beforeExecution();
-608lock.readLock().lock();
-609try {
-610  notifyFlushRequest(region, 
emergencyFlush);
-611  FlushResult flushResult = 
region.flushcache(forceFlushAllStores, false, tracker);
-612  boolean shouldCompact = 
flushResult.isCompactionNeeded();
-613  // We just want to check the size
-614  boolean shouldSplit = 
region.checkSplit() != null;
-615  if (shouldSplit) {
-616
this.server.compactSplitThread.requestSplit(region);
-617  } else if (shouldCompact) {
-618
server.compactSplitThread.requestSystemCompaction(region, 
Thread.currentThread().getName());
-619  }
-620} catch (DroppedSnapshotException ex) 
{
-621  // Cache flush can fail in a few 
places. If it fails in a critical
-622  // section, we get a 
DroppedSnapshotException and a replay of wal
-623  // is required. Currently the only 
way to do this is a restart of
-624  // the server. Abort because hdfs 
is probably bad (HBASE-644 is a case
-625  // where hdfs was bad but passed 
the hdfs check).
-626  server.abort("Replay of WAL 
required. Forcing server shutdown", ex);
-627  return false;
-628} catch (IOException ex) {
-629  ex = ex instanceof RemoteException 
? ((RemoteException) ex).unwrapRemoteException() : ex;
-630  LOG.error(
-631"Cache flush failed"
-632+ (region != null ? (" for 
region " +
-633
Bytes.toStringBinary(region.getRegionInfo().getRegionName()))
-634  : ""), ex);
-635  if (!server.checkFileSystem()) {
-636return false;
-637



[28/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/DefaultMemStore.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/DefaultMemStore.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/DefaultMemStore.html
index 411bb17..24bd483 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/DefaultMemStore.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/DefaultMemStore.html
@@ -105,127 +105,122 @@
 097return new 
MemStoreSnapshot(this.snapshotId, this.snapshot);
 098  }
 099
-100  /**
-101   * On flush, how much memory we will 
clear from the active cell set.
-102   *
-103   * @return size of data that is going 
to be flushed from active set
-104   */
-105  @Override
-106  public MemStoreSize getFlushableSize() 
{
-107MemStoreSize snapshotSize = 
getSnapshotSize();
-108return snapshotSize.getDataSize() 
> 0 ? snapshotSize
-109: new 
MemStoreSize(active.getMemStoreSize());
-110  }
-111
-112  @Override
-113  protected long keySize() {
-114return this.active.keySize();
-115  }
-116
-117  @Override
-118  protected long heapSize() {
-119return this.active.heapSize();
-120  }
-121
-122  @Override
-123  /*
-124   * Scanners are ordered from 0 (oldest) 
to newest in increasing order.
-125   */
-126  public List 
getScanners(long readPt) throws IOException {
-127List list = 
new ArrayList<>();
-128addToScanners(active, readPt, 
list);
-129
addToScanners(snapshot.getAllSegments(), readPt, list);
-130return list;
-131  }
-132
-133  @Override
-134  protected List 
getSegments() throws IOException {
-135List list = new 
ArrayList<>(2);
-136list.add(this.active);
-137list.add(this.snapshot);
-138return list;
-139  }
-140
-141  /**
-142   * @param cell Find the row that comes 
after this one.  If null, we return the
-143   * first.
-144   * @return Next row or null if none 
found.
-145   */
-146  Cell getNextRow(final Cell cell) {
-147return getLowest(
-148getNextRow(cell, 
this.active.getCellSet()),
-149getNextRow(cell, 
this.snapshot.getCellSet()));
-150  }
-151
-152  @Override public void 
updateLowestUnflushedSequenceIdInWAL(boolean onlyIfMoreRecent) {
-153  }
-154
-155  @Override
-156  public MemStoreSize size() {
-157return new 
MemStoreSize(active.getMemStoreSize());
-158  }
-159
-160  /**
-161   * Check whether anything need to be 
done based on the current active set size
-162   * Nothing need to be done for the 
DefaultMemStore
-163   */
-164  @Override
-165  protected void checkActiveSize() {
-166return;
-167  }
-168
-169  @Override
-170  public long preFlushSeqIDEstimation() 
{
-171return HConstants.NO_SEQNUM;
-172  }
-173
-174  @Override public boolean isSloppy() {
-175return false;
-176  }
-177
-178  /**
-179   * Code to help figure if our 
approximation of object heap sizes is close
-180   * enough.  See hbase-900.  Fills 
memstores then waits so user can heap
-181   * dump and bring up resultant hprof in 
something like jprofiler which
-182   * allows you get 'deep size' on 
objects.
-183   * @param args main args
-184   */
-185  public static void main(String [] args) 
{
-186RuntimeMXBean runtime = 
ManagementFactory.getRuntimeMXBean();
-187LOG.info("vmName=" + 
runtime.getVmName() + ", vmVendor=" +
-188  runtime.getVmVendor() + ", 
vmVersion=" + runtime.getVmVersion());
-189LOG.info("vmInputArguments=" + 
runtime.getInputArguments());
-190DefaultMemStore memstore1 = new 
DefaultMemStore();
-191// TODO: x32 vs x64
-192final int count = 1;
-193byte [] fam = Bytes.toBytes("col");
-194byte [] qf = Bytes.toBytes("umn");
-195byte [] empty = new byte[0];
-196MemStoreSizing memstoreSizing = new 
MemStoreSizing();
+100  @Override
+101  public MemStoreSize getFlushableSize() 
{
+102MemStoreSize mss = 
getSnapshotSize();
+103return mss.getDataSize() > 0? mss: 
this.active.getMemStoreSize();
+104  }
+105
+106  @Override
+107  protected long keySize() {
+108return this.active.getDataSize();
+109  }
+110
+111  @Override
+112  protected long heapSize() {
+113return this.active.getHeapSize();
+114  }
+115
+116  @Override
+117  /*
+118   * Scanners are ordered from 0 (oldest) 
to newest in increasing order.
+119   */
+120  public List 
getScanners(long readPt) throws IOException {
+121List list = 
new ArrayList<>();
+122addToScanners(active, readPt, 
list);
+123
addToScanners(snapshot.getAllSegments(), readPt, list);
+124return list;
+125  }
+126
+127  @Override
+128  protected List 
getSegments() throws IOException {
+129List list = new 
ArrayList<>(2);
+130list.add(this.active);
+131list.add(this.snapshot);
+132return list



[06/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.LimitFields.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.LimitFields.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.LimitFields.html
index 03a0b2a..cabb570 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.LimitFields.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/ScannerContext.LimitFields.html
@@ -561,209 +561,206 @@
 553
 554/**
 555 * Set all fields together.
-556 * @param batch
-557 * @param sizeScope
-558 * @param dataSize
-559 */
-560void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
-561LimitScope timeScope, long time) 
{
-562  setBatch(batch);
-563  setSizeScope(sizeScope);
-564  setDataSize(dataSize);
-565  setHeapSize(heapSize);
-566  setTimeScope(timeScope);
-567  setTime(time);
-568}
-569
-570int getBatch() {
-571  return this.batch;
-572}
-573
-574void setBatch(int batch) {
-575  this.batch = batch;
-576}
-577
-578/**
-579 * @param checkerScope
-580 * @return true when the limit can be 
enforced from the scope of the checker
-581 */
-582boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
-583  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
-584}
-585
-586long getDataSize() {
-587  return this.dataSize;
-588}
-589
-590long getHeapSize() {
-591  return this.heapSize;
-592}
-593
-594void setDataSize(long dataSize) {
-595  this.dataSize = dataSize;
-596}
-597
-598void setHeapSize(long heapSize) {
-599  this.heapSize = heapSize;
-600}
-601
-602/**
-603 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
-604 */
-605LimitScope getSizeScope() {
-606  return this.sizeScope;
-607}
-608
-609/**
-610 * Change the scope in which the size 
limit is enforced
-611 */
-612void setSizeScope(LimitScope scope) 
{
-613  this.sizeScope = scope;
-614}
-615
-616/**
-617 * @param checkerScope
-618 * @return true when the limit can be 
enforced from the scope of the checker
-619 */
-620boolean 
canEnforceSizeLimitFromScope(LimitScope checkerScope) {
-621  return 
this.sizeScope.canEnforceLimitFromScope(checkerScope);
-622}
-623
-624long getTime() {
-625  return this.time;
-626}
-627
-628void setTime(long time) {
-629  this.time = time;
-630}
-631
-632/**
-633 * @return {@link LimitScope} 
indicating scope in which the time limit is enforced
-634 */
-635LimitScope getTimeScope() {
-636  return this.timeScope;
-637}
-638
-639/**
-640 * Change the scope in which the time 
limit is enforced
-641 */
-642void setTimeScope(LimitScope scope) 
{
-643  this.timeScope = scope;
-644}
-645
-646/**
-647 * @param checkerScope
-648 * @return true when the limit can be 
enforced from the scope of the checker
-649 */
-650boolean 
canEnforceTimeLimitFromScope(LimitScope checkerScope) {
-651  return 
this.timeScope.canEnforceLimitFromScope(checkerScope);
-652}
-653
-654@Override
-655public String toString() {
-656  StringBuilder sb = new 
StringBuilder();
-657  sb.append("{");
+556 */
+557void setFields(int batch, LimitScope 
sizeScope, long dataSize, long heapSize,
+558LimitScope timeScope, long time) 
{
+559  setBatch(batch);
+560  setSizeScope(sizeScope);
+561  setDataSize(dataSize);
+562  setHeapSize(heapSize);
+563  setTimeScope(timeScope);
+564  setTime(time);
+565}
+566
+567int getBatch() {
+568  return this.batch;
+569}
+570
+571void setBatch(int batch) {
+572  this.batch = batch;
+573}
+574
+575/**
+576 * @param checkerScope
+577 * @return true when the limit can be 
enforced from the scope of the checker
+578 */
+579boolean 
canEnforceBatchLimitFromScope(LimitScope checkerScope) {
+580  return 
LimitScope.BETWEEN_CELLS.canEnforceLimitFromScope(checkerScope);
+581}
+582
+583long getDataSize() {
+584  return this.dataSize;
+585}
+586
+587long getHeapSize() {
+588  return this.heapSize;
+589}
+590
+591void setDataSize(long dataSize) {
+592  this.dataSize = dataSize;
+593}
+594
+595void setHeapSize(long heapSize) {
+596  this.heapSize = heapSize;
+597}
+598
+599/**
+600 * @return {@link LimitScope} 
indicating scope in which the size limit is enforced
+601 */
+602LimitScope getSizeScope() {
+603  return this.sizeScope;
+604}
+605
+606/**
+607 * Change the scope in which the



[46/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
index 6a4d81b..b5ca7c9 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/CompactingMemStore.html
@@ -321,7 +321,8 @@ extends 
 MemStoreSize
 getFlushableSize()
-On flush, how much memory we will clear.
+Flush will first clear out the data in snapshot if any (It 
will take a second flush
+ invocation to clear the current Cell set).
 
 
 
@@ -478,7 +479,7 @@ extends AbstractMemStore
-add,
 add,
 addToScanners,
 addToScanners,
 clearSnapshot,
 dump,
 getActive, getComparator,
 getConfiguration,
 getLowest,
 getNextRow,
 getSnapshot,
 getSnapshotSize,
 getSnapshotSizing,
 resetActive,
 timeOfOldestEdit,
 toString,
 upsert
+add,
 add,
 addToScanners,
 addToScanners,
 clearSnapshot,
 dump,
 getActive, getComparator,
 getConfiguration,
 getLowest,
 getNextRow,
 getSnapshot,
 getSnapshotSize,
 resetActive,
 timeOfOldestEdit,
 toString,
 upsert
 
 
 
@@ -777,11 +778,14 @@ extends 
 
 getFlushableSize
-public MemStoreSize getFlushableSize()
-On flush, how much memory we will clear.
+public MemStoreSize getFlushableSize()
+Description copied from 
interface: MemStore
+Flush will first clear out the data in snapshot if any (It 
will take a second flush
+ invocation to clear the current Cell set). If snapshot is empty, current
+ Cell set will be flushed.
 
 Returns:
-size of data that is going to be flushed
+On flush, how much memory we will clear.
 
 
 
@@ -791,7 +795,7 @@ extends 
 
 keySize
-protected long keySize()
+protected long keySize()
 
 Specified by:
 keySize in
 class AbstractMemStore
@@ -806,7 +810,7 @@ extends 
 
 heapSize
-protected long heapSize()
+protected long heapSize()
 
 Specified by:
 heapSize in
 class AbstractMemStore
@@ -822,7 +826,7 @@ extends 
 
 updateLowestUnflushedSequenceIdInWAL
-public void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater)
+public void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater)
 Description copied from 
class: AbstractMemStore
 Updates the wal with the lowest sequence id (oldest entry) 
that is still in memory
 
@@ -840,7 +844,7 @@ extends 
 
 startReplayingFromWAL
-public void startReplayingFromWAL()
+public void startReplayingFromWAL()
 This message intends to inform the MemStore that next 
coming updates
  are going to be part of the replaying edits from WAL
 
@@ -851,7 +855,7 @@ extends 
 
 stopReplayingFromWAL
-public void stopReplayingFromWAL()
+public void stopReplayingFromWAL()
 This message intends to inform the MemStore that the 
replaying edits from WAL
  are done
 
@@ -862,7 +866,7 @@ extends 
 
 getSegments
-protected https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true";
 title="class or interface in java.util">List getSegments()
+protected https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true";
 title="class or interface in java.util">List getSegments()
 
 Specified by:
 getSegments in
 class AbstractMemStore
@@ -877,7 +881,7 @@ extends 
 
 setCompositeSnapshot
-public void setCompositeSnapshot(boolean useCompositeSnapshot)
+public void setCompositeSnapshot(boolean useCompositeSnapshot)
 
 
 
@@ -886,7 +890,7 @@ extends 
 
 swapCompactedSegments
-public boolean swapCompactedSegments(VersionedSegmentsList versionedList,
+public boolean swapCompactedSegments(VersionedSegmentsList versionedList,
  ImmutableSegment result,
  boolean merge)
 
@@ -897,7 +901,7 @@ extends 
 
 flattenOneSegment
-public void flattenOneSegment(long requesterVersion,
+public void flattenOneSegment(long requesterVersion,
   MemStoreCompactionStrategy.Action action)
 
 Parameters:
@@ -913,7 +917,7 @@ extends 
 
 setIndexType
-void setIndexType(CompactingMemStore.IndexType type)
+void setIndexType(CompactingMemStore.IndexType type)
 
 
 
@@ -922,7 +926,7 @@ extends 
 
 getIndexType
-public CompactingMemStore.IndexType getIndexType()
+public CompactingMemStore.IndexType getIndexType()
 
 
 
@@ -931,7 +935,7 @@ extends 
 
 hasImmutableSegments
-public boolean hasImmutableSegments()
+public boolean hasImmutableSegments()
 
 
 
@@ -940,7 +944,7 @@ extends 
 
 getImmutableSegments
-public VersionedSegmentsList getImmutableSegments()
+public VersionedSegmentsList getImmutableSegments()
 
 
 
@@ -949,7 +953,7 @@ extends 
 
 getSmallestReadPoint
-public long getSmallestReadPoint()
+public long getSmallestReadPoint()
 
 
 
@@ -958,7 +962,7 @@ extends 
 
 getStore
-public HStore getStore()
+public HStore getStore()
 
 
 
@@ -967,7 +971,7 @@ extends 
 
 ge



[38/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.html
 
b/devapidocs/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.html
new file mode 100644
index 000..41011f5
--- /dev/null
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.html
@@ -0,0 +1,489 @@
+http://www.w3.org/TR/html4/loose.dtd";>
+
+
+
+
+
+NonThreadSafeMemStoreSizing (Apache HBase 3.0.0-SNAPSHOT API)
+
+
+
+
+
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10};
+var tabs = {65535:["t0","All Methods"],2:["t2","Instance 
Methods"],8:["t4","Concrete Methods"]};
+var altColor = "altColor";
+var rowColor = "rowColor";
+var tableTab = "tableTab";
+var activeTableTab = "activeTableTab";
+
+
+JavaScript is disabled on your browser.
+
+
+
+
+
+Skip navigation links
+
+
+
+
+Overview
+Package
+Class
+Use
+Tree
+Deprecated
+Index
+Help
+
+
+
+
+Prev Class
+Next Class
+
+
+Frames
+No Frames
+
+
+All Classes
+
+
+
+
+
+
+
+Summary: 
+Nested | 
+Field | 
+Constr | 
+Method
+
+
+Detail: 
+Field | 
+Constr | 
+Method
+
+
+
+
+
+
+
+
+org.apache.hadoop.hbase.regionserver
+Class 
NonThreadSafeMemStoreSizing
+
+
+
+https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">java.lang.Object
+
+
+org.apache.hadoop.hbase.regionserver.NonThreadSafeMemStoreSizing
+
+
+
+
+
+
+
+All Implemented Interfaces:
+MemStoreSizing
+
+
+
+@InterfaceAudience.Private
+class NonThreadSafeMemStoreSizing
+extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+implements MemStoreSizing
+Accounting of current heap and data sizes.
+ NOT THREAD SAFE.
+ Use in a 'local' context only where just a single-thread is updating. No 
concurrency!
+ Used, for example, when summing all Cells in a single batch where result is 
then applied to the
+ Store.
+
+See Also:
+ThreadSafeMemStoreSizing
+
+
+
+
+
+
+
+
+
+
+
+
+Field Summary
+
+Fields 
+
+Modifier and Type
+Field and Description
+
+
+private long
+dataSize 
+
+
+private long
+heapSize 
+
+
+private long
+offHeapSize 
+
+
+
+
+
+
+Fields inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+DUD
+
+
+
+
+
+
+
+
+Constructor Summary
+
+Constructors 
+
+Constructor and Description
+
+
+NonThreadSafeMemStoreSizing() 
+
+
+NonThreadSafeMemStoreSizing(long dataSize,
+   long heapSize,
+   long offHeapSize) 
+
+
+NonThreadSafeMemStoreSizing(MemStoreSize mss) 
+
+
+
+
+
+
+
+
+
+Method Summary
+
+All Methods Instance Methods Concrete Methods 
+
+Modifier and Type
+Method and Description
+
+
+long
+getDataSize() 
+
+
+long
+getHeapSize() 
+
+
+MemStoreSize
+getMemStoreSize() 
+
+
+long
+getOffHeapSize() 
+
+
+long
+incMemStoreSize(long dataSizeDelta,
+   long heapSizeDelta,
+   long offHeapSizeDelta) 
+
+
+https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String
+toString() 
+
+
+
+
+
+
+Methods inherited from class java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#clone--";
 title="class or interface in java.lang">clone, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#equals-java.lang.Object-";
 title="class or interface in java.lang">equals, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#finalize--";
 title="class or interface in java.lang">finalize, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#getClass--";
 title="class or interface in java.lang">getClass, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#hashCode--";
 title="class or interface in java.lang">hashCode, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#notify--";
 title="class or interface in java.lang">notify, https://docs.oracle.com/javase/8/docs/api/ja
 va/lang/Object.html?is-external=true#notifyAll--" title="class or interface in 
java.lang">notifyAll, https://docs.oracle.com/javase/



[20/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ObservedExceptionsInBatch.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ObservedExceptionsInBatch.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ObservedExceptionsInBatch.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ObservedExceptionsInBatch.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ObservedExceptionsInBatch.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required



[14/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.WriteState.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.WriteState.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.WriteState.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.WriteState.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.WriteState.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesForStores 
getRegionSe



[45/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.html
 
b/devapidocs/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.html
index a4b2e8a..b81f11b 100644
--- 
a/devapidocs/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.html
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.html
@@ -117,6 +117,10 @@ var activeTableTab = "activeTableTab";
 
 
 
+
+All Implemented Interfaces:
+MemStoreSizing
+
 
 
 @InterfaceAudience.Private
@@ -165,7 +169,14 @@ extends Segment
-FIXED_OVERHEAD,
 minSequenceId,
 segmentSize,
 tagsPresent,
 timeRangeTracker
+FIXED_OVERHEAD,
 memStoreSizing,
 minSequenceId,
 tagsPresent,
 timeRangeTracker
+
+
+
+
+
+Fields inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+DUD
 
 
 
@@ -243,72 +254,72 @@ extends 
 long
-getMinSequenceId() 
+getDataSize() 
 
 
+long
+getHeapSize() 
+
+
+long
+getMinSequenceId() 
+
+
 int
 getNumOfSegments() 
 
-
+
 KeyValueScanner
 getScanner(long readPoint)
 Creates the scanner for the given read point
 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true";
 title="class or interface in java.util">List
 getScanners(long readPoint) 
 
-
+
 TimeRangeTracker
 getTimeRangeTracker() 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/SortedSet.html?is-external=true";
 title="class or interface in java.util">SortedSet
 headSet(Cell firstKeyOnRow) 
 
-
-long
-heapSize() 
-
-
-void
-incScannerCount() 
-
 
-protected void
-incSize(long delta,
-   long heapOverhead,
-   long offHeapOverhead)
+long
+incMemStoreSize(long delta,
+   long heapOverhead,
+   long offHeapOverhead)
 Updates the heap size counter of the segment by the given 
delta
 
 
 
+void
+incScannerCount() 
+
+
 protected long
 indexEntrySize() 
 
-
+
 protected void
 internalAdd(Cell cell,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
-
+
 boolean
 isEmpty() 
 
-
+
 boolean
 isTagsPresent() 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true";
 title="class or interface in java.util">Iterator
 iterator() 
 
-
-long
-keySize() 
-
 
 Cell
 last() 
@@ -349,7 +360,7 @@ extends updateMetaInfo(Cell cellToAdd,
   boolean succ,
   boolean mslabUsed,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
 
 
@@ -364,7 +375,7 @@ extends Segment
-getCellLength,
 getComparator,
 getMemStoreLAB,
 getMemStoreSize,
 heapSizeChange,
 indexEntryOffHeapSize,
 indexEntryOnHeapSize,
 offHeapSize,
  offHeapSizeChange,
 updateMetaInfo
+getCellLength,
 getComparator,
 getMemStoreLAB,
 getMemStoreSize,
 getOffHeapSize,
 heapSizeChange,
 indexEntryOffHeapSize,
 indexEntryOnHeapSiz
 e, offHeapSizeChange,
 updateMetaInfo
 
 
 
@@ -373,6 +384,13 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
 https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#clone--";
 title="class or interface in java.lang">clone, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#equals-java.lang.Object-";
 title="class or interface in java.lang">equals, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#finalize--";
 title="class or interface in java.lang">finalize, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#getClass--";
 title="class or interface in java.lang">getClass, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#hashCode--";
 title="class or interface in java.lang">hashCode, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#notify--";
 title="class or interface in java.lang">notify, https://docs.oracle.com/javase/8/docs/api/ja
 va/lang/Object.html?is-external=true#notifyAll--" title="class or interface in 
java.lang">notifyAll, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#wait--";
 title="class or interface in java.lang">wait, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#wait-long-";
 title="class or interface in java.lang">wait, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#wait-long-int-";
 title="class or interface in java.lang">wait
 
+
+
+
+
+Methods inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+decMemStoreSize,
 decMemStoreSize,
 incMemStoreSize
+
 
 
 
@@ -645,49 +663,57 @@ extends 
+
 
 
 
 
-keySize
-public long keySize(



[12/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
index cca21a9..2f8a48b 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.StoreFlusherImpl.html
@@ -2280,7 +2280,7 @@
 2272  this.cacheFlushCount = 
snapshot.getCellsCount();
 2273  this.cacheFlushSize = 
snapshot.getDataSize();
 2274  committedFiles = new 
ArrayList<>(1);
-2275  return new 
MemStoreSize(snapshot.getMemStoreSize());
+2275  return 
snapshot.getMemStoreSize();
 2276}
 2277
 2278@Override

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.html
index cca21a9..2f8a48b 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HStore.html
@@ -2280,7 +2280,7 @@
 2272  this.cacheFlushCount = 
snapshot.getCellsCount();
 2273  this.cacheFlushSize = 
snapshot.getDataSize();
 2274  committedFiles = new 
ArrayList<>(1);
-2275  return new 
MemStoreSize(snapshot.getMemStoreSize());
+2275  return 
snapshot.getMemStoreSize();
 2276}
 2277
 2278@Override

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStore.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStore.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStore.html
index 541a093..651511e 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStore.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStore.html
@@ -58,96 +58,95 @@
 050  void clearSnapshot(long id) throws 
UnexpectedStateException;
 051
 052  /**
-053   * On flush, how much memory we will 
clear.
-054   * Flush will first clear out the data 
in snapshot if any (It will take a second flush
-055   * invocation to clear the current Cell 
set). If snapshot is empty, current
-056   * Cell set will be flushed.
-057   *
-058   * @return size of data that is going 
to be flushed
-059   */
-060  MemStoreSize getFlushableSize();
-061
-062  /**
-063   * Return the size of the snapshot(s) 
if any
-064   * @return size of the memstore 
snapshot
-065   */
-066  MemStoreSize getSnapshotSize();
-067
-068  /**
-069   * Write an update
-070   * @param cell
-071   * @param memstoreSizing The delta in 
memstore size will be passed back via this.
-072   *This will include both data 
size and heap overhead delta.
-073   */
-074  void add(final Cell cell, 
MemStoreSizing memstoreSizing);
-075
-076  /**
-077   * Write the updates
-078   * @param cells
-079   * @param memstoreSizing The delta in 
memstore size will be passed back via this.
-080   *This will include both data 
size and heap overhead delta.
-081   */
-082  void add(Iterable cells, 
MemStoreSizing memstoreSizing);
-083
-084  /**
-085   * @return Oldest timestamp of all the 
Cells in the MemStore
-086   */
-087  long timeOfOldestEdit();
-088
-089  /**
-090   * Update or insert the specified 
cells.
-091   * 

-092   * For each Cell, insert into MemStore. 
This will atomically upsert the value for that
-093   * row/family/qualifier. If a Cell did 
already exist, it will then be removed.
-094   * 

-095   * Currently the memstoreTS is kept at 
0 so as each insert happens, it will be immediately
-096   * visible. May want to change this so 
it is atomic across all KeyValues.
-097   * 

-098   * This is called under row lock, so 
Get operations will still see updates atomically. Scans will
-099   * only see each KeyValue update as 
atomic.
-100   * @param cells
-101   * @param readpoint readpoint below 
which we can safely remove duplicate Cells.
-102   * @param memstoreSizing The delta in 
memstore size will be passed back via this.
-103   *This will include both data 
size and heap overhead delta.
-104   */
-105  void upsert(Iterable cells, 
long readpoint, MemStoreSizing memstoreSizing);
-106
-107  /**
-108   * @return scanner over the memstore. 
This might include scanner over the snapshot when one is
-109   * present.
-110   */
-111  List 
getScanners(long readPt) throws IOException;
-112
-113  /**
-114 



[34/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/package-summary.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/package-summary.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/package-summary.html
index 3fc380e..67c417f 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/package-summary.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/package-summary.html
@@ -200,215 +200,221 @@
 
 
 
+MemStoreSizing
+
+Accounting of current heap and data sizes.
+
+
+
 MetricsHeapMemoryManagerSource
 
 This interface will be implemented by a MetricsSource that 
will export metrics from
  HeapMemoryManager in RegionServer into the hadoop metrics system.
 
 
-
+
 MetricsRegionAggregateSource
 
 This interface will be implemented by a MetricsSource that 
will export metrics from
  multiple regions into the hadoop metrics system.
 
 
-
+
 MetricsRegionServerQuotaSource
 
 A collection of exposed metrics for space quotas from an 
HBase RegionServer.
 
 
-
+
 MetricsRegionServerSource
 
 Interface for classes that expose metrics about the 
regionserver.
 
 
-
+
 MetricsRegionServerSourceFactory
 
 Interface of a factory to create Metrics Sources used 
inside of regionservers.
 
 
-
+
 MetricsRegionServerWrapper
 
 This is the interface that will expose RegionServer 
information to hadoop1/hadoop2
  implementations of the MetricsRegionServerSource.
 
 
-
+
 MetricsRegionSource
 
 This interface will be implemented to allow single regions 
to push metrics into
  MetricsRegionAggregateSource that will in turn push data to the Hadoop 
metrics system.
 
 
-
+
 MetricsRegionWrapper
 
 Interface of class that will wrap an HRegion and export 
numbers so they can be
  used in MetricsRegionSource
 
 
-
+
 MetricsTableAggregateSource
 
 This interface will be implemented by a MetricsSource that 
will export metrics from
  multiple regions of a table into the hadoop metrics system.
 
 
-
+
 MetricsTableLatencies
 
 Latency metrics for a specific table in a 
RegionServer.
 
 
-
+
 MetricsTableSource
 
 This interface will be implemented to allow region server 
to push table metrics into
  MetricsRegionAggregateSource that will in turn push data to the Hadoop 
metrics system.
 
 
-
+
 MetricsTableWrapperAggregate
 
 Interface of class that will wrap a MetricsTableSource and 
export numbers so they can be
  used in MetricsTableSource
 
 
-
+
 MutableOnlineRegions
 
 Interface to Map of online regions.
 
 
-
+
 OnlineRegions
 
 Provides read-only access to the Regions presently online 
on the
  current RegionServer
 
 
-
+
 Region
 
 Region is a subset of HRegion with operations required for 
the Coprocessors.
 
 
-
+
 Region.RowLock
 
 Row lock held by a given thread.
 
 
-
+
 RegionScanner
 
 RegionScanner describes iterators over rows in an 
HRegion.
 
 
-
+
 RegionServerServices
 
 A curated subset of services provided by HRegionServer.
 
 
-
+
 ReplicationService
 
 Gateway to Cluster Replication.
 
 
-
+
 ReplicationSinkService
 
 A sink for a replication stream has to expose this 
service.
 
 
-
+
 ReplicationSourceService
 
 A source for a replication stream has to expose this 
service.
 
 
-
+
 RowProcessor
 Deprecated
 As of release 2.0.0, this 
will be removed in HBase 3.0.0.
 
 
-
+
 RpcSchedulerFactory
 
 A factory class that constructs an RpcScheduler.
 
 
-
+
 RSRpcServices.LogDelegate
  
 
-
+
 ScanOptions
 
 This class gives you the ability to change the max versions 
and TTL options before opening a
  scanner for a Store.
 
 
-
+
 SequenceId
 
 Interface which abstracts implementations on log sequenceId 
assignment
 
 
-
+
 Shipper
 
 This interface denotes a scanner as one which can ship 
cells.
 
 
-
+
 ShipperListener
 
 Implementors of this interface are the ones who needs to do 
some action when the
  Shipper.shipped()
 is called
 
 
-
+
 SplitLogWorker.TaskExecutor
 
 Objects implementing this interface actually do the task 
that has been
  acquired by a SplitLogWorker.
 
 
-
+
 Store
 
 Interface for objects that hold a column family in a 
Region.
 
 
-
+
 StoreConfigInformation
 
 A more restricted interface for HStore.
 
 
-
+
 StoreFile
 
 An interface to describe a store data file.
 
 
-
+
 StoreFileManager
 
 Manages the store files and basic metadata about that that 
determines the logical structure
  (e.g.
 
 
-
+
 StoreFlushContext
 
 A package protected interface for a store flushing.
@@ -990,46 +996,40 @@
 
 MemStoreSize
 
-Reports the data size part and total heap space occupied by 
the MemStore.
+Data structure of three longs.
 
 
 
-MemStoreSizing
-
-Accounting of current heap and data sizes.
-
-
-
 MemStoreSnapshot
 
 Holds details of the snapshot taken on a MemStore.
 
 
-
+
 MetricsHeapMemoryManager
 
 This class is for maintaining the various regio



[50/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/checkstyle-aggregate.html
--
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 70ea00a..014a328 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase – Checkstyle Results
 
@@ -281,10 +281,10 @@
  Warnings
  Errors
 
-3610
+3612
 0
 0
-15878
+15873
 
 Files
 
@@ -6432,7 +6432,7 @@
 org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
 0
 0
-6
+4
 
 org/apache/hadoop/hbase/regionserver/CompactionTool.java
 0
@@ -6877,7 +6877,7 @@
 org/apache/hadoop/hbase/regionserver/ScannerContext.java
 0
 0
-12
+9
 
 org/apache/hadoop/hbase/regionserver/ScannerIdGenerator.java
 0
@@ -10234,7 +10234,7 @@
 
 
 http://checkstyle.sourceforge.net/config_blocks.html#NeedBraces";>NeedBraces
-1934
+1932
  Error
 
 coding
@@ -10327,12 +10327,12 @@
 http://checkstyle.sourceforge.net/config_javadoc.html#JavadocTagContinuationIndentation";>JavadocTagContinuationIndentation
 
 offset: "2"
-797
+783
  Error
 
 
 http://checkstyle.sourceforge.net/config_javadoc.html#NonEmptyAtclauseDescription";>NonEmptyAtclauseDescription
-3822
+3833
  Error
 
 misc
@@ -14541,7 +14541,7 @@
 
  Error
 javadoc
-JavadocTagContinuationIndentation
+NonEmptyAtclauseDescription
 Javadoc comment at column 26 has parse error. Missed HTML close tag 'arg'. 
Sometimes it means that close tag missed for one of previous tags.
 44
 
@@ -15177,7 +15177,7 @@
 
  Error
 javadoc
-JavadocTagContinuationIndentation
+NonEmptyAtclauseDescription
 Javadoc comment at column 4 has parse error. Missed HTML close tag 'pre'. 
Sometimes it means that close tag missed for one of previous tags.
 59
 
@@ -16932,7 +16932,7 @@
 
  Error
 javadoc
-JavadocTagContinuationIndentation
+NonEmptyAtclauseDescription
 Javadoc comment at column 19 has parse error. Details: no viable 
alternative at input '\n   *   List



[27/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.Visitor.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.Visitor.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.Visitor.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.Visitor.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.Visitor.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level



[42/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
index 5e16d7d..cbb0f00 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 
 
 
-public static class HRegion.RowLockImpl
+public static class HRegion.RowLockImpl
 extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
 implements Region.RowLock
 Class used to represent a lock on a row.
@@ -226,7 +226,7 @@ implements 
 
 context
-private final HRegion.RowLockContext context
+private final HRegion.RowLockContext context
 
 
 
@@ -235,7 +235,7 @@ implements 
 
 lock
-private final https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/locks/Lock.html?is-external=true";
 title="class or interface in java.util.concurrent.locks">Lock lock
+private final https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/locks/Lock.html?is-external=true";
 title="class or interface in java.util.concurrent.locks">Lock lock
 
 
 
@@ -252,7 +252,7 @@ implements 
 
 RowLockImpl
-public RowLockImpl(HRegion.RowLockContext context,
+public RowLockImpl(HRegion.RowLockContext context,
https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/locks/Lock.html?is-external=true";
 title="class or interface in 
java.util.concurrent.locks">Lock lock)
 
 
@@ -270,7 +270,7 @@ implements 
 
 getLock
-public https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/locks/Lock.html?is-external=true";
 title="class or interface in java.util.concurrent.locks">Lock getLock()
+public https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/locks/Lock.html?is-external=true";
 title="class or interface in java.util.concurrent.locks">Lock getLock()
 
 
 
@@ -279,7 +279,7 @@ implements 
 
 getContext
-public HRegion.RowLockContext getContext()
+public HRegion.RowLockContext getContext()
 
 
 
@@ -288,7 +288,7 @@ implements 
 
 release
-public void release()
+public void release()
 Description copied from 
interface: Region.RowLock
 Release the given lock.  If there are no remaining locks 
held by the current thread
  then unlock the row and allow other threads to acquire the lock.
@@ -304,7 +304,7 @@ implements 
 
 toString
-public https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String toString()
+public https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String toString()
 
 Overrides:
 https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#toString--";
 title="class or interface in java.lang">toString in 
class https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object






[07/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreSize.html 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
index cdc0ca3..f434f88 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
@@ -27,109 +27,99 @@
 019
 020import 
org.apache.hadoop.hbase.HBaseInterfaceAudience;
 021import 
org.apache.yetus.audience.InterfaceAudience;
-022
-023/**
-024 * Reports the data size part and total 
heap space occupied by the MemStore.
-025 * Read-only.
+022/**
+023 * Data structure of three longs.
+024 * Convenient package in which to carry 
current state of three counters.
+025 * 
Immutable!
 026 * @see MemStoreSizing
 027 */
 028@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
 029public class MemStoreSize {
-030  // MemStore size tracks 3 sizes:
-031  // (1) data size: the aggregated size 
of all key-value not including meta data such as
-032  // index, time range etc.
-033  // (2) heap size: the aggregated size 
of all data that is allocated on-heap including all
-034  // key-values that reside on-heap and 
the metadata that resides on-heap
-035  // (3) off-heap size: the aggregated 
size of all data that is allocated off-heap including all
-036  // key-values that reside off-heap and 
the metadata that resides off-heap
-037  //
-038  // 3 examples to illustrate their 
usage:
-039  // Consider a store with 100MB of 
key-values allocated on-heap and 20MB of metadata allocated
-040  // on-heap. The counters are <100MB, 
120MB, 0>, respectively.
-041  // Consider a store with 100MB of 
key-values allocated off-heap and 20MB of metadata
-042  // allocated on-heap (e.g, CAM index). 
The counters are <100MB, 20MB, 100MB>, respectively.
-043  // Consider a store with 100MB of 
key-values from which 95MB are allocated off-heap and 5MB
-044  // are allocated on-heap (e.g., due to 
upserts) and 20MB of metadata from which 15MB allocated
-045  // off-heap (e.g, CCM index) and 5MB 
allocated on-heap (e.g, CSLM index in active).
-046  // The counters are <100MB, 10MB, 
110MB>, respectively.
-047
-048  /**
-049   *'dataSize' tracks the Cell's data 
bytes size alone (Key bytes, value bytes). A cell's data can
-050   * be in on heap or off heap area 
depending on the MSLAB and its configuration to be using on heap
-051   * or off heap LABs
-052   */
-053  protected volatile long dataSize;
-054
-055  /** 'heapSize' tracks all Cell's heap 
size occupancy. This will include Cell POJO heap overhead.
-056   * When Cells in on heap area, this 
will include the cells data size as well.
-057   */
-058  protected volatile long heapSize;
-059
-060  /** off-heap size: the aggregated size 
of all data that is allocated off-heap including all
-061   * key-values that reside off-heap and 
the metadata that resides off-heap
-062   */
-063  protected volatile long offHeapSize;
-064
-065  public MemStoreSize() {
-066this(0L, 0L, 0L);
-067  }
-068
-069  public MemStoreSize(long dataSize, long 
heapSize, long offHeapSize) {
-070this.dataSize = dataSize;
-071this.heapSize = heapSize;
-072this.offHeapSize = offHeapSize;
-073  }
-074
-075  protected MemStoreSize(MemStoreSize 
memStoreSize) {
-076this.dataSize = 
memStoreSize.dataSize;
-077this.heapSize = 
memStoreSize.heapSize;
-078this.offHeapSize = 
memStoreSize.offHeapSize;
-079  }
-080  public boolean isEmpty() {
-081return this.dataSize == 0 && 
this.heapSize == 0 && this.offHeapSize == 0;
+030  /**
+031   *'dataSize' tracks the Cell's data 
bytes size alone (Key bytes, value bytes). A cell's data can
+032   * be in on heap or off heap area 
depending on the MSLAB and its configuration to be using on
+033   * heap or off heap LABs
+034   */
+035  private final long dataSize;
+036
+037  /**'getHeapSize' tracks all Cell's heap 
size occupancy. This will include Cell POJO heap overhead.
+038   * When Cells in on heap area, this 
will include the cells data size as well.
+039   */
+040  private final long heapSize;
+041
+042  /** off-heap size: the aggregated size 
of all data that is allocated off-heap including all
+043   * key-values that reside off-heap and 
the metadata that resides off-heap
+044   */
+045  private final long offHeapSize;
+046
+047  /**
+048   * Package private constructor.
+049   */
+050  MemStoreSize() {
+051this(0L, 0L, 0L);
+052  }
+053
+054  /**
+055   * Package private constructor.
+056   */
+057  MemStoreSize(long dataSize, long 
heapSize, long offHeapSize) {
+058this.dataSize = dataSize;
+059this.heapSize = heapSize;
+060this.offHeapSize = offHeapSize;
+061  }
+062
+063  /**
+064   * Package private constructor.
+065   */
+066  M



[29/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionPipeline.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionPipeline.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionPipeline.html
index 2ab9e4d..c18869f 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionPipeline.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactionPipeline.html
@@ -31,323 +31,327 @@
 023import java.util.LinkedList;
 024import java.util.List;
 025
-026import 
org.apache.yetus.audience.InterfaceAudience;
-027import org.slf4j.Logger;
-028import org.slf4j.LoggerFactory;
-029import 
org.apache.hadoop.hbase.util.Bytes;
-030import 
org.apache.hadoop.hbase.util.ClassSize;
-031
-032/**
-033 * The compaction pipeline of a {@link 
CompactingMemStore}, is a FIFO queue of segments.
-034 * It supports pushing a segment at the 
head of the pipeline and removing a segment from the
-035 * tail when it is flushed to disk.
-036 * It also supports swap method to allow 
the in-memory compaction swap a subset of the segments
-037 * at the tail of the pipeline with a new 
(compacted) one. This swap succeeds only if the version
-038 * number passed with the list of 
segments to swap is the same as the current version of the
-039 * pipeline.
-040 * Essentially, there are two methods 
which can change the structure of the pipeline: pushHead()
-041 * and swap(), the later is used both by 
a flush to disk and by an in-memory compaction.
-042 * The pipeline version is updated by 
swap(); it allows to identify conflicting operations at the
-043 * suffix of the pipeline.
-044 *
-045 * The synchronization model is 
copy-on-write. Methods which change the structure of the
-046 * pipeline (pushHead(), 
flattenOneSegment() and swap()) apply their changes in the context of a
-047 * lock. They also make a read-only copy 
of the pipeline's list. Read methods read from a
-048 * read-only copy. If a read method 
accesses the read-only copy more than once it makes a local
-049 * copy of it to ensure it accesses the 
same copy.
-050 *
-051 * The methods getVersionedList(), 
getVersionedTail(), and flattenOneSegment() are also
-052 * protected by a lock since they need to 
have a consistent (atomic) view of the pipeline list
-053 * and version number.
-054 */
-055@InterfaceAudience.Private
-056public class CompactionPipeline {
-057  private static final Logger LOG = 
LoggerFactory.getLogger(CompactionPipeline.class);
-058
-059  public final static long FIXED_OVERHEAD 
= ClassSize
-060  .align(ClassSize.OBJECT + (3 * 
ClassSize.REFERENCE) + Bytes.SIZEOF_LONG);
-061  public final static long DEEP_OVERHEAD 
= FIXED_OVERHEAD + (2 * ClassSize.LINKEDLIST);
-062
-063  private final RegionServicesForStores 
region;
-064  private final 
LinkedList pipeline = new LinkedList<>();
-065  // The list is volatile to avoid 
reading a new allocated reference before the c'tor is executed
-066  private volatile 
LinkedList readOnlyCopy = new LinkedList<>();
-067  // Version is volatile to ensure it is 
atomically read when not using a lock
-068  private volatile long version = 0;
-069
-070  public 
CompactionPipeline(RegionServicesForStores region) {
-071this.region = region;
-072  }
-073
-074  public boolean pushHead(MutableSegment 
segment) {
-075ImmutableSegment immutableSegment = 
SegmentFactory.instance().
-076
createImmutableSegment(segment);
-077synchronized (pipeline){
-078  boolean res = 
addFirst(immutableSegment);
-079  readOnlyCopy = new 
LinkedList<>(pipeline);
-080  return res;
-081}
-082  }
-083
-084  public VersionedSegmentsList 
getVersionedList() {
-085synchronized (pipeline){
-086  return new 
VersionedSegmentsList(readOnlyCopy, version);
-087}
-088  }
-089
-090  public VersionedSegmentsList 
getVersionedTail() {
-091synchronized (pipeline){
-092  List 
segmentList = new ArrayList<>();
-093  if(!pipeline.isEmpty()) {
-094segmentList.add(0, 
pipeline.getLast());
-095  }
-096  return new 
VersionedSegmentsList(segmentList, version);
-097}
-098  }
-099
-100  /**
-101   * Swaps the versioned list at the tail 
of the pipeline with a new segment.
-102   * Swapping only if there were no 
changes to the suffix of the list since the version list was
-103   * created.
-104   * @param versionedList suffix of the 
pipeline to be replaced can be tail or all the pipeline
-105   * @param segment new segment to 
replace the suffix. Can be null if the suffix just needs to be
-106   *removed.
-107   * @param closeSuffix whether to close 
the suffix (to release memory), as part of swapping it out
-108   *During index merge op this 
will be false and for compaction it will be



[41/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.html
index 7ec6aa5..e2dd09e 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.html
@@ -18,7 +18,7 @@
 catch(err) {
 }
 //-->
-var methods = 
{"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10,"i38":10,"i39":10,"i40":10,"i41":9,"i42":9,"i43":9,"i44":9,"i45":10,"i46":10,"i47":10,"i48":10,"i49":10,"i50":10,"i51":10,"i52":10,"i53":10,"i54":10,"i55":10,"i56":10,"i57":10,"i58":10,"i59":10,"i60":9,"i61":10,"i62":10,"i63":10,"i64":10,"i65":10,"i66":10,"i67":10,"i68":10,"i69":10,"i70":10,"i71":10,"i72":10,"i73":10,"i74":10,"i75":10,"i76":10,"i77":10,"i78":10,"i79":10,"i80":10,"i81":10,"i82":10,"i83":10,"i84":10,"i85":10,"i86":10,"i87":10,"i88":10,"i89":10,"i90":10,"i91":10,"i92":9,"i93":10,"i94":10,"i95":10,"i96":10,"i97":10,"i98":10,"i99":10,"i100":10,"i101":10,"i102":10,"i103":10,"i104":10,"i105":9,"i106":10,"i107":10,"i108":10,"i109":10
 
,"i110":10,"i111":10,"i112":41,"i113":41,"i114":10,"i115":10,"i116":10,"i117":10,"i118":10,"i119":10,"i120":10,"i121":10,"i122":10,"i123":10,"i124":10,"i125":10,"i126":10,"i127":10,"i128":10,"i129":10,"i130":10,"i131":10,"i132":10,"i133":10,"i134":10,"i135":10,"i136":10,"i137":10,"i138":9,"i139":10,"i140":10,"i141":10,"i142":10,"i143":10,"i144":10,"i145":10,"i146":42,"i147":10,"i148":10,"i149":10,"i150":10,"i151":10,"i152":10,"i153":10,"i154":10,"i155":10,"i156":10,"i157":10,"i158":10,"i159":10,"i160":10,"i161":10,"i162":10,"i163":10,"i164":10,"i165":10,"i166":10,"i167":10,"i168":9,"i169":10,"i170":10,"i171":10,"i172":10,"i173":10,"i174":10,"i175":10,"i176":10,"i177":9,"i178":10,"i179":10,"i180":9,"i181":9,"i182":9,"i183":9,"i184":9,"i185":9,"i186":9,"i187":9,"i188":9,"i189":9,"i190":10,"i191":10,"i192":10,"i193":10,"i194":10,"i195":10,"i196":10,"i197":10,"i198":10,"i199":9,"i200":10,"i201":10,"i202":10,"i203":10,"i204":10,"i205":10,"i206":10,"i207":10,"i208":10,"i209":10,"i210":10,
 
"i211":10,"i212":10,"i213":10,"i214":10,"i215":10,"i216":10,"i217":10,"i218":10,"i219":10,"i220":10,"i221":10,"i222":10,"i223":10,"i224":10,"i225":10,"i226":10,"i227":10,"i228":10,"i229":10,"i230":9,"i231":9,"i232":10,"i233":10,"i234":10,"i235":10,"i236":10,"i237":10,"i238":10,"i239":10,"i240":10,"i241":10,"i242":10,"i243":9,"i244":10,"i245":10,"i246":10,"i247":10,"i248":10,"i249":10,"i250":10,"i251":10,"i252":10,"i253":10,"i254":10,"i255":10,"i256":10,"i257":9,"i258":10,"i259":10,"i260":10,"i261":10};
+var methods = 
{"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10,"i38":10,"i39":10,"i40":10,"i41":9,"i42":9,"i43":9,"i44":9,"i45":10,"i46":10,"i47":10,"i48":10,"i49":10,"i50":10,"i51":10,"i52":10,"i53":10,"i54":10,"i55":10,"i56":10,"i57":10,"i58":10,"i59":10,"i60":10,"i61":9,"i62":10,"i63":10,"i64":10,"i65":10,"i66":10,"i67":10,"i68":10,"i69":10,"i70":10,"i71":10,"i72":10,"i73":10,"i74":10,"i75":10,"i76":10,"i77":10,"i78":10,"i79":10,"i80":10,"i81":10,"i82":10,"i83":10,"i84":10,"i85":10,"i86":10,"i87":10,"i88":10,"i89":10,"i90":10,"i91":10,"i92":10,"i93":9,"i94":10,"i95":10,"i96":10,"i97":10,"i98":10,"i99":10,"i100":10,"i101":10,"i102":10,"i103":10,"i104":10,"i105":10,"i106":9,"i107":10,"i108":10,"i109":10
 
,"i110":10,"i111":10,"i112":10,"i113":41,"i114":41,"i115":10,"i116":10,"i117":10,"i118":10,"i119":10,"i120":10,"i121":10,"i122":10,"i123":10,"i124":10,"i125":10,"i126":10,"i127":10,"i128":10,"i129":10,"i130":10,"i131":10,"i132":10,"i133":10,"i134":10,"i135":10,"i136":10,"i137":10,"i138":10,"i139":9,"i140":10,"i141":10,"i142":10,"i143":10,"i144":10,"i145":10,"i146":10,"i147":10,"i148":42,"i149":10,"i150":10,"i151":10,"i152":10,"i153":10,"i154":10,"i155":10,"i156":10,"i157":10,"i158":10,"i159":10,"i160":10,"i161":10,"i162":10,"i163":10,"i164":10,"i165":10,"i166":10,"i167":10,"i168":10,"i169":10,"i170":9,"i171":10,"i172":10,"i173":10,"i174":10,"i175":10,"i176":10,"i177":10,"i178":10,"i179":9,"i180":10,"i181":10,"i182":9,"i183":9,"i184":9,"i185":9,"i186":9,"i187":9,"i188":9,"i189":9,"i190":9,"i191":9,"i192":10,"i193":10,"i194":10,"i195":10,"i196":10,"i197":10,"i198":10,"i199":10,"i200":10,"i201":9,"i202":10,"i203":10,"i204":10,"i



[47/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
index abd4ac3..7d20fe1 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
@@ -18,7 +18,7 @@
 catch(err) {
 }
 //-->
-var methods = 
{"i0":10,"i1":10,"i2":9,"i3":9,"i4":6,"i5":10,"i6":9,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":6,"i14":10,"i15":10,"i16":10,"i17":6,"i18":10,"i19":6,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":6,"i26":10,"i27":10};
+var methods = 
{"i0":10,"i1":10,"i2":9,"i3":9,"i4":6,"i5":10,"i6":9,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":6,"i14":10,"i15":10,"i16":6,"i17":10,"i18":6,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":6,"i25":10,"i26":10};
 var tabs = {65535:["t0","All Methods"],1:["t1","Static 
Methods"],2:["t2","Instance Methods"],4:["t3","Abstract 
Methods"],8:["t4","Concrete Methods"]};
 var altColor = "altColor";
 var rowColor = "rowColor";
@@ -214,14 +214,14 @@ implements 
 void
 add(Cell cell,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Write an update
 
 
 
 void
 add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Write the updates
 
 
@@ -294,24 +294,20 @@ implements 
-(package private) MemStoreSizing
-getSnapshotSizing() 
-
-
 protected abstract long
 heapSize() 
 
-
+
 private void
 internalAdd(Cell toAdd,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
-
+
 protected abstract long
 keySize() 
 
-
+
 private Cell
 maybeCloneWithAllocator(Cell cell,
boolean forceCloneOfBigCell)
@@ -321,39 +317,39 @@ implements 
+
 protected void
 resetActive() 
 
-
+
 private void
 setOldestEditTimeToNow() 
 
-
+
 long
 timeOfOldestEdit() 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String
 toString() 
 
-
+
 abstract void
 updateLowestUnflushedSequenceIdInWAL(boolean onlyIfMoreRecent)
 Updates the wal with the lowest sequence id (oldest entry) 
that is still in memory
 
 
-
+
 private void
 upsert(Cell cell,
   long readpoint,
-  MemStoreSizing memstoreSizing) 
+  MemStoreSizing memstoreSizing) 
 
-
+
 void
 upsert(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
   long readpoint,
-  MemStoreSizing memstoreSizing)
+  MemStoreSizing memstoreSizing)
 Update or insert the specified cells.
 
 
@@ -550,7 +546,7 @@ implements 
 add
 public void add(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
-MemStoreSizing memstoreSizing)
+MemStoreSizing memstoreSizing)
 Description copied from 
interface: MemStore
 Write the updates
 
@@ -568,7 +564,7 @@ implements 
 add
 public void add(Cell cell,
-MemStoreSizing memstoreSizing)
+MemStoreSizing memstoreSizing)
 Description copied from 
interface: MemStore
 Write an update
 
@@ -596,7 +592,7 @@ implements upsert(https://docs.oracle.com/javase/8/docs/api/java/lang/Iterable.html?is-external=true";
 title="class or interface in java.lang">Iterable cells,
long readpoint,
-   MemStoreSizing memstoreSizing)
+   MemStoreSizing memstoreSizing)
 Description copied from 
interface: MemStore
 Update or insert the specified cells.
  
@@ -670,22 +666,13 @@ implements 
-
-
-
-
-getSnapshotSizing
-MemStoreSizing getSnapshotSizing()
-
-
 
 
 
 
 
 toString
-public https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String toString()
+public https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String toString()
 
 Overrides:
 https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#toString--";
 title="class or interface in java.lang">toString in 
class https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
@@ -698,7 +685,7 @@ implements 
 
 getConfiguration
-protected org.apache.hadoop.conf.Configuration getConfiguration()
+protected org.apache.hadoop.conf.Configuration getConfiguration()
 
 
 
@@ -707,7 +6



[21/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.MutationBatchOperation.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.MutationBatchOperation.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.MutationBatchOperation.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.MutationBatchOperation.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.MutationBatchOperation.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level



[39/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/MemStoreSize.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
index a40bb14..b6b595f 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/MemStoreSize.html
@@ -50,7 +50,7 @@ var activeTableTab = "activeTableTab";
 
 
 Prev Class
-Next Class
+Next Class
 
 
 Frames
@@ -107,20 +107,17 @@ var activeTableTab = "activeTableTab";
 
 
 
-
-Direct Known Subclasses:
-MemStoreSizing
-
 
 
 @InterfaceAudience.LimitedPrivate(value="Coprocesssor")
 public class MemStoreSize
 extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
-Reports the data size part and total heap space occupied by 
the MemStore.
- Read-only.
+Data structure of three longs.
+ Convenient package in which to carry current state of three counters.
+ Immutable!
 
 See Also:
-MemStoreSizing
+MemStoreSizing
 
 
 
@@ -141,19 +138,19 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 Field and Description
 
 
-protected long
+private long
 dataSize
 'dataSize' tracks the Cell's data bytes size alone (Key 
bytes, value bytes).
 
 
 
-protected long
+private long
 heapSize
-'heapSize' tracks all Cell's heap size occupancy.
+'getHeapSize' tracks all Cell's heap size occupancy.
 
 
 
-protected long
+private long
 offHeapSize
 off-heap size: the aggregated size of all data that is 
allocated off-heap including all
  key-values that reside off-heap and the metadata that resides off-heap
@@ -171,22 +168,24 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 Constructors 
 
-Modifier
-Constructor and Description
+Constructor and Description
 
 
- 
-MemStoreSize() 
+MemStoreSize()
+Package private constructor.
+
 
 
- 
-MemStoreSize(long dataSize,
+MemStoreSize(long dataSize,
 long heapSize,
-long offHeapSize) 
+long offHeapSize)
+Package private constructor.
+
 
 
-protected 
-MemStoreSize(MemStoreSize memStoreSize) 
+MemStoreSize(MemStoreSize memStoreSize)
+Package private constructor.
+
 
 
 
@@ -259,10 +258,10 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 dataSize
-protected volatile long dataSize
+private final long dataSize
 'dataSize' tracks the Cell's data bytes size alone (Key 
bytes, value bytes). A cell's data can
- be in on heap or off heap area depending on the MSLAB and its configuration 
to be using on heap
- or off heap LABs
+ be in on heap or off heap area depending on the MSLAB and its configuration 
to be using on
+ heap or off heap LABs
 
 
 
@@ -271,8 +270,8 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 heapSize
-protected volatile long heapSize
-'heapSize' tracks all Cell's heap size occupancy. This will 
include Cell POJO heap overhead.
+private final long heapSize
+'getHeapSize' tracks all Cell's heap size occupancy. This 
will include Cell POJO heap overhead.
  When Cells in on heap area, this will include the cells data size as 
well.
 
 
@@ -282,7 +281,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 offHeapSize
-protected volatile long offHeapSize
+private final long offHeapSize
 off-heap size: the aggregated size of all data that is 
allocated off-heap including all
  key-values that reside off-heap and the metadata that resides off-heap
 
@@ -301,7 +300,8 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 MemStoreSize
-public MemStoreSize()
+MemStoreSize()
+Package private constructor.
 
 
 
@@ -310,9 +310,10 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 MemStoreSize
-public MemStoreSize(long dataSize,
-long heapSize,
-long offHeapSize)
+MemStoreSize(long dataSize,
+ long heapSize,
+ long offHeapSize)
+Package private constructor.
 
 
 
@@ -321,7 +322,8 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 MemStoreSize
-protected MemStoreSize(MemStoreSize memStoreSize)
+MemStoreSize(MemStoreSize memStoreSize)
+Package private constructor.
 
 
 
@@ -338,7 +340,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 isEmpty
-public boolean isEmpty()
+public boolean isEmpty()
 
 
 
@@ -347,7 +349,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 getDataSize
-public long getDataSize()
+public long getDataSize()
 
 
 
@@ -356,7 +358,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 getHeapSize
-public long getHeapSize()
+public long getHeapSize()
 
 
 
@@ -365,7 +367,7 @@



[01/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site a8dd4f308 -> 92a26cfbe


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/testdevapidocs/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.ReadOwnWritesTester.html
--
diff --git 
a/testdevapidocs/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.ReadOwnWritesTester.html
 
b/testdevapidocs/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.ReadOwnWritesTester.html
index b08ee76..057351b 100644
--- 
a/testdevapidocs/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.ReadOwnWritesTester.html
+++ 
b/testdevapidocs/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.ReadOwnWritesTester.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 
 
 
-private static class TestDefaultMemStore.ReadOwnWritesTester
+private static class TestDefaultMemStore.ReadOwnWritesTester
 extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true";
 title="class or interface in java.lang">Thread
 
 
@@ -270,7 +270,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 NUM_TRIES
-static final int NUM_TRIES
+static final int NUM_TRIES
 
 See Also:
 Constant
 Field Values
@@ -283,7 +283,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 row
-final byte[] row
+final byte[] row
 
 
 
@@ -292,7 +292,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 f
-final byte[] f
+final byte[] f
 
 
 
@@ -301,7 +301,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 q1
-final byte[] q1
+final byte[] q1
 
 
 
@@ -310,7 +310,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 mvcc
-final org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl
 mvcc
+final org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl
 mvcc
 
 
 
@@ -319,7 +319,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 memstore
-final org.apache.hadoop.hbase.regionserver.MemStore memstore
+final org.apache.hadoop.hbase.regionserver.MemStore memstore
 
 
 
@@ -328,7 +328,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 caughtException
-https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicReference.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicReferenceThrowable> caughtException
+https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicReference.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicReferenceThrowable> caughtException
 
 
 
@@ -345,7 +345,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 ReadOwnWritesTester
-public ReadOwnWritesTester(int id,
+public ReadOwnWritesTester(int id,

org.apache.hadoop.hbase.regionserver.MemStore memstore,

org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl mvcc,
https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicReference.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicReferenceThrowable> caughtException)
@@ -365,7 +365,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 run
-public void run()
+public void run()
 
 Specified by:
 https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true#run--";
 title="class or interface in java.lang">run in 
interface https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true";
 title="class or interface in java.lang">Runnable
@@ -380,7 +380,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
 
 
 internalRun
-private void internalRun()
+private void internalRun()
   throws https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true";
 title="class or interface in java.io">IOException
 
 Throws:






[23/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesForStores 
getReg



[43/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
index 8eb6446..5499ca5 100644
--- 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
@@ -163,7 +163,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 storeFlushCtxs 
 
 
-(package private) MemStoreSizing
+(package private) MemStoreSizing
 totalFlushableSize 
 
 
@@ -197,7 +197,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
   long startTime,
   long flushSeqId,
   long flushedSeqId,
-  MemStoreSizing totalFlushableSize) 
+  MemStoreSizing totalFlushableSize) 
 
 
 (package private)
@@ -207,7 +207,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
   long startTime,
   long flushSeqId,
   long flushedSeqId,
-  MemStoreSizing totalFlushableSize)
+  MemStoreSizing totalFlushableSize)
 Constructs a successful prepare flush result
 
 
@@ -321,7 +321,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 totalFlushableSize
-final MemStoreSizing totalFlushableSize
+final MemStoreSizing totalFlushableSize
 
 
 
@@ -355,7 +355,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
long startTime,
long flushSeqId,
long flushedSeqId,
-   MemStoreSizing totalFlushableSize)
+   MemStoreSizing totalFlushableSize)
 Constructs a successful prepare flush result
 
 
@@ -372,7 +372,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
long startTime,
long flushSeqId,
long flushedSeqId,
-   MemStoreSizing totalFlushableSize)
+   MemStoreSizing totalFlushableSize)
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
index 72c83ed..f668456 100644
--- 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.RegionScannerImpl.html
@@ -121,7 +121,7 @@ var activeTableTab = "activeTableTab";
 
 
 
-class HRegion.RegionScannerImpl
+class HRegion.RegionScannerImpl
 extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
 implements RegionScanner, Shipper, RpcCallback
 RegionScannerImpl is used to combine scanners from multiple 
Stores (aka column families).
@@ -425,7 +425,7 @@ implements 
 
 storeHeap
-KeyValueHeap storeHeap
+KeyValueHeap storeHeap
 
 
 
@@ -434,7 +434,7 @@ implements 
 
 joinedHeap
-KeyValueHeap joinedHeap
+KeyValueHeap joinedHeap
 Heap of key-values that are not essential for the provided 
filters and are thus read
  on demand, if on-demand column family loading is enabled.
 
@@ -445,7 +445,7 @@ implements 
 
 joinedContinuationRow
-protected Cell joinedContinuationRow
+protected Cell joinedContinuationRow
 If the joined heap data gathering is interrupted due to 
scan limits, this will
  contain the row for which we are populating the values.
 
@@ -456,7 +456,7 @@ implements 
 
 filterClosed
-private boolean filterClosed
+private boolean filterClosed
 
 
 
@@ -465,7 +465,7 @@ implements 
 
 stopRow
-protected final byte[] stopRow
+protected final byte[] stopRow
 
 
 
@@ -474,7 +474,7 @@ implements 
 
 includeStopRow
-protected final boolean includeStopRow
+protected final boolean includeStopRow
 
 
 
@@ -483,7 +483,7 @@ implements 
 
 region
-protected final HRegion region
+protected final HRegion region
 
 
 
@@ -492,7 +492,7 @@ implements 
 
 comparator
-protected final CellComparator comparator
+protected final CellComparator comparator
 
 
 
@@ -501,7 +501,7 @@ implements 
 
 readPt
-private final long readPt
+private final long readPt
 
 
 
@@ -510,7 +510,7 @@ implements 
 
 maxResultSize
-private final long maxResultSize
+private final long maxResultSize
 
 
 
@@ -519,7 +519,7 @@ implements 
 
 defaultScannerContext
-private final ScannerContext defaultScannerContext
+priva



[44/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
index 79df939..3ddf42b 100644
--- 
a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 
 
 
-private abstract static class HRegion.BatchOperation
+private abstract static class HRegion.BatchOperation
 extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
 Class that tracks the progress of a batch operations, 
accumulating status codes and tracking
  the index at which processing is proceeding. These batch operations may get 
split into
@@ -240,7 +240,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 protected void
 applyFamilyMapToMemStore(https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true";
 title="class or interface in java.util">MapList> familyMap,
-MemStoreSizing memstoreAccounting)
+MemStoreSizing memstoreAccounting)
 Atomically apply the given map of family->edits to the 
memstore.
 
 
@@ -415,7 +415,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 operations
-protected final T[] operations
+protected final T[] operations
 
 
 
@@ -424,7 +424,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 retCodeDetails
-protected final OperationStatus[] retCodeDetails
+protected final OperationStatus[] retCodeDetails
 
 
 
@@ -433,7 +433,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 walEditsFromCoprocessors
-protected final WALEdit[] walEditsFromCoprocessors
+protected final WALEdit[] walEditsFromCoprocessors
 
 
 
@@ -442,7 +442,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 familyCellMaps
-protected final https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true";
 title="class or interface in java.util">MapList>[] familyCellMaps
+protected final https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true";
 title="class or interface in java.util">MapList>[] familyCellMaps
 
 
 
@@ -451,7 +451,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 region
-protected final HRegion region
+protected final HRegion region
 
 
 
@@ -460,7 +460,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 nextIndexToProcess
-protected int nextIndexToProcess
+protected int nextIndexToProcess
 
 
 
@@ -469,7 +469,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 observedExceptions
-protected final HRegion.ObservedExceptionsInBatch observedExceptions
+protected final HRegion.ObservedExceptionsInBatch observedExceptions
 
 
 
@@ -478,7 +478,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 durability
-protected Durability durability
+protected Durability durability
 
 
 
@@ -487,7 +487,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 atomic
-protected boolean atomic
+protected boolean atomic
 
 
 
@@ -506,7 +506,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 BatchOperation
-public BatchOperation(HRegion region,
+public BatchOperation(HRegion region,
   T[] operations)
 
 
@@ -524,7 +524,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 visitBatchOperations
-public void visitBatchOperations(boolean pendingOnly,
+public void visitBatchOperations(boolean pendingOnly,
  int lastIndexExclusive,
  HRegion.BatchOperation.Visitor visitor)
   throws https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true";
 title="class or interface in java.io">IOException
@@ -541,7 +541,7 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 getMutation
-public abstract Mutation getMutation(int index)
+public abstract Mutation getMutation(int index)
 
 
 
@@ -550,7 +550,7 @@ exte



[26/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.BatchOperation.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesFo



[31/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html
index d4390be..1037b84 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html
@@ -163,12 +163,12 @@
 155   */
 156  @Override
 157  public MemStoreSize size() {
-158MemStoreSizing memstoreSizing = new 
MemStoreSizing();
+158MemStoreSizing memstoreSizing = new 
NonThreadSafeMemStoreSizing();
 159
memstoreSizing.incMemStoreSize(active.getMemStoreSize());
 160for (Segment item : 
pipeline.getSegments()) {
 161  
memstoreSizing.incMemStoreSize(item.getMemStoreSize());
 162}
-163return memstoreSizing;
+163return 
memstoreSizing.getMemStoreSize();
 164  }
 165
 166  /**
@@ -224,369 +224,365 @@
 216return new 
MemStoreSnapshot(snapshotId, this.snapshot);
 217  }
 218
-219  /**
-220   * On flush, how much memory we will 
clear.
-221   * @return size of data that is going 
to be flushed
-222   */
-223  @Override
-224  public MemStoreSize getFlushableSize() 
{
-225MemStoreSizing snapshotSizing = 
getSnapshotSizing();
-226if (snapshotSizing.getDataSize() == 
0) {
-227  // if snapshot is empty the tail of 
the pipeline (or everything in the memstore) is flushed
-228  if (compositeSnapshot) {
-229snapshotSizing = 
pipeline.getPipelineSizing();
-230
snapshotSizing.incMemStoreSize(active.getMemStoreSize());
-231  } else {
-232snapshotSizing = 
pipeline.getTailSizing();
-233  }
-234}
-235return snapshotSizing.getDataSize() 
> 0 ? snapshotSizing
-236: new 
MemStoreSize(active.getMemStoreSize());
-237  }
-238
-239  @Override
-240  protected long keySize() {
-241// Need to consider keySize of all 
segments in pipeline and active
-242long k = this.active.keySize();
-243for (Segment segment : 
this.pipeline.getSegments()) {
-244  k += segment.keySize();
-245}
-246return k;
-247  }
-248
-249  @Override
-250  protected long heapSize() {
-251// Need to consider heapOverhead of 
all segments in pipeline and active
-252long h = this.active.heapSize();
-253for (Segment segment : 
this.pipeline.getSegments()) {
-254  h += segment.heapSize();
-255}
-256return h;
-257  }
-258
-259  @Override
-260  public void 
updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater) {
-261long minSequenceId = 
pipeline.getMinSequenceId();
-262if(minSequenceId != Long.MAX_VALUE) 
{
-263  byte[] encodedRegionName = 
getRegionServices().getRegionInfo().getEncodedNameAsBytes();
-264  byte[] familyName = 
getFamilyNameInBytes();
-265  WAL WAL = 
getRegionServices().getWAL();
-266  if (WAL != null) {
-267
WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater);
-268  }
-269}
-270  }
-271
-272  /**
-273   * This message intends to inform the 
MemStore that next coming updates
-274   * are going to be part of the 
replaying edits from WAL
-275   */
-276  @Override
-277  public void startReplayingFromWAL() {
-278inWalReplay = true;
-279  }
-280
-281  /**
-282   * This message intends to inform the 
MemStore that the replaying edits from WAL
-283   * are done
-284   */
-285  @Override
-286  public void stopReplayingFromWAL() {
-287inWalReplay = false;
-288  }
-289
-290  // the getSegments() method is used for 
tests only
-291  @VisibleForTesting
-292  @Override
-293  protected List 
getSegments() {
-294List 
pipelineList = pipeline.getSegments();
-295List list = new 
ArrayList<>(pipelineList.size() + 2);
-296list.add(this.active);
-297list.addAll(pipelineList);
-298
list.addAll(this.snapshot.getAllSegments());
-299
-300return list;
-301  }
-302
-303  // the following three methods allow to 
manipulate the settings of composite snapshot
-304  public void 
setCompositeSnapshot(boolean useCompositeSnapshot) {
-305this.compositeSnapshot = 
useCompositeSnapshot;
-306  }
-307
-308  public boolean 
swapCompactedSegments(VersionedSegmentsList versionedList, ImmutableSegment 
result,
-309  boolean merge) {
-310// last true stands for updating the 
region size
-311return pipeline.swap(versionedList, 
result, !merge, true);
-312  }
-313
-314  /**
-315   * @param requesterVersion The caller 
must hold the VersionedList of the pipeline
-316   *   with version taken 
earlier. This version must be passed as a parameter here.
-317   *   The flattening happens 
only if versions match.
-318   */
-319  public v



[32/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.InMemoryFlushRunnable.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.InMemoryFlushRunnable.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.InMemoryFlushRunnable.html
index d4390be..1037b84 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.InMemoryFlushRunnable.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/CompactingMemStore.InMemoryFlushRunnable.html
@@ -163,12 +163,12 @@
 155   */
 156  @Override
 157  public MemStoreSize size() {
-158MemStoreSizing memstoreSizing = new 
MemStoreSizing();
+158MemStoreSizing memstoreSizing = new 
NonThreadSafeMemStoreSizing();
 159
memstoreSizing.incMemStoreSize(active.getMemStoreSize());
 160for (Segment item : 
pipeline.getSegments()) {
 161  
memstoreSizing.incMemStoreSize(item.getMemStoreSize());
 162}
-163return memstoreSizing;
+163return 
memstoreSizing.getMemStoreSize();
 164  }
 165
 166  /**
@@ -224,369 +224,365 @@
 216return new 
MemStoreSnapshot(snapshotId, this.snapshot);
 217  }
 218
-219  /**
-220   * On flush, how much memory we will 
clear.
-221   * @return size of data that is going 
to be flushed
-222   */
-223  @Override
-224  public MemStoreSize getFlushableSize() 
{
-225MemStoreSizing snapshotSizing = 
getSnapshotSizing();
-226if (snapshotSizing.getDataSize() == 
0) {
-227  // if snapshot is empty the tail of 
the pipeline (or everything in the memstore) is flushed
-228  if (compositeSnapshot) {
-229snapshotSizing = 
pipeline.getPipelineSizing();
-230
snapshotSizing.incMemStoreSize(active.getMemStoreSize());
-231  } else {
-232snapshotSizing = 
pipeline.getTailSizing();
-233  }
-234}
-235return snapshotSizing.getDataSize() 
> 0 ? snapshotSizing
-236: new 
MemStoreSize(active.getMemStoreSize());
-237  }
-238
-239  @Override
-240  protected long keySize() {
-241// Need to consider keySize of all 
segments in pipeline and active
-242long k = this.active.keySize();
-243for (Segment segment : 
this.pipeline.getSegments()) {
-244  k += segment.keySize();
-245}
-246return k;
-247  }
-248
-249  @Override
-250  protected long heapSize() {
-251// Need to consider heapOverhead of 
all segments in pipeline and active
-252long h = this.active.heapSize();
-253for (Segment segment : 
this.pipeline.getSegments()) {
-254  h += segment.heapSize();
-255}
-256return h;
-257  }
-258
-259  @Override
-260  public void 
updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater) {
-261long minSequenceId = 
pipeline.getMinSequenceId();
-262if(minSequenceId != Long.MAX_VALUE) 
{
-263  byte[] encodedRegionName = 
getRegionServices().getRegionInfo().getEncodedNameAsBytes();
-264  byte[] familyName = 
getFamilyNameInBytes();
-265  WAL WAL = 
getRegionServices().getWAL();
-266  if (WAL != null) {
-267
WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater);
-268  }
-269}
-270  }
-271
-272  /**
-273   * This message intends to inform the 
MemStore that next coming updates
-274   * are going to be part of the 
replaying edits from WAL
-275   */
-276  @Override
-277  public void startReplayingFromWAL() {
-278inWalReplay = true;
-279  }
-280
-281  /**
-282   * This message intends to inform the 
MemStore that the replaying edits from WAL
-283   * are done
-284   */
-285  @Override
-286  public void stopReplayingFromWAL() {
-287inWalReplay = false;
-288  }
-289
-290  // the getSegments() method is used for 
tests only
-291  @VisibleForTesting
-292  @Override
-293  protected List 
getSegments() {
-294List 
pipelineList = pipeline.getSegments();
-295List list = new 
ArrayList<>(pipelineList.size() + 2);
-296list.add(this.active);
-297list.addAll(pipelineList);
-298
list.addAll(this.snapshot.getAllSegments());
-299
-300return list;
-301  }
-302
-303  // the following three methods allow to 
manipulate the settings of composite snapshot
-304  public void 
setCompositeSnapshot(boolean useCompositeSnapshot) {
-305this.compositeSnapshot = 
useCompositeSnapshot;
-306  }
-307
-308  public boolean 
swapCompactedSegments(VersionedSegmentsList versionedList, ImmutableSegment 
result,
-309  boolean merge) {
-310// last true stands for updating the 
region size
-311return pipeline.swap(versionedList, 
result, !merge, true);
-312  }
-313
-314  /**
-315   * @param requesterVersion The caller 
must hold the VersionedList of the pipeline
-316   *   with version taken 
earlier. This version must be passed as a parameter here.
-317   *   The flatteni



[19/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.PrepareFlushResult.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  pub



[16/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockContext.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockContext.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockContext.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockContext.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockContext.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesFo



[36/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.html
--
diff --git 
a/devapidocs/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.html
new file mode 100644
index 000..b497649
--- /dev/null
+++ 
b/devapidocs/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.html
@@ -0,0 +1,487 @@
+http://www.w3.org/TR/html4/loose.dtd";>
+
+
+
+
+
+ThreadSafeMemStoreSizing (Apache HBase 3.0.0-SNAPSHOT API)
+
+
+
+
+
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10};
+var tabs = {65535:["t0","All Methods"],2:["t2","Instance 
Methods"],8:["t4","Concrete Methods"]};
+var altColor = "altColor";
+var rowColor = "rowColor";
+var tableTab = "tableTab";
+var activeTableTab = "activeTableTab";
+
+
+JavaScript is disabled on your browser.
+
+
+
+
+
+Skip navigation links
+
+
+
+
+Overview
+Package
+Class
+Use
+Tree
+Deprecated
+Index
+Help
+
+
+
+
+Prev Class
+Next Class
+
+
+Frames
+No Frames
+
+
+All Classes
+
+
+
+
+
+
+
+Summary: 
+Nested | 
+Field | 
+Constr | 
+Method
+
+
+Detail: 
+Field | 
+Constr | 
+Method
+
+
+
+
+
+
+
+
+org.apache.hadoop.hbase.regionserver
+Class 
ThreadSafeMemStoreSizing
+
+
+
+https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">java.lang.Object
+
+
+org.apache.hadoop.hbase.regionserver.ThreadSafeMemStoreSizing
+
+
+
+
+
+
+
+All Implemented Interfaces:
+MemStoreSizing
+
+
+
+@InterfaceAudience.Private
+class ThreadSafeMemStoreSizing
+extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+implements MemStoreSizing
+Accounting of current heap and data sizes.
+ Thread-safe. Many threads can do updates against this single instance.
+
+See Also:
+NonThreadSafeMemStoreSizing,
 
+MemStoreSize
+
+
+
+
+
+
+
+
+
+
+
+
+Field Summary
+
+Fields 
+
+Modifier and Type
+Field and Description
+
+
+private https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicLong
+dataSize 
+
+
+private https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicLong
+heapSize 
+
+
+private https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true";
 title="class or interface in 
java.util.concurrent.atomic">AtomicLong
+offHeapSize 
+
+
+
+
+
+
+Fields inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+DUD
+
+
+
+
+
+
+
+
+Constructor Summary
+
+Constructors 
+
+Constructor and Description
+
+
+ThreadSafeMemStoreSizing() 
+
+
+ThreadSafeMemStoreSizing(long dataSize,
+long heapSize,
+long offHeapSize) 
+
+
+ThreadSafeMemStoreSizing(MemStoreSize mss) 
+
+
+
+
+
+
+
+
+
+Method Summary
+
+All Methods Instance Methods Concrete Methods 
+
+Modifier and Type
+Method and Description
+
+
+long
+getDataSize() 
+
+
+long
+getHeapSize() 
+
+
+MemStoreSize
+getMemStoreSize() 
+
+
+long
+getOffHeapSize() 
+
+
+long
+incMemStoreSize(long dataSizeDelta,
+   long heapSizeDelta,
+   long offHeapSizeDelta) 
+
+
+https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true";
 title="class or interface in java.lang">String
+toString() 
+
+
+
+
+
+
+Methods inherited from class java.lang.https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#clone--";
 title="class or interface in java.lang">clone, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#equals-java.lang.Object-";
 title="class or interface in java.lang">equals, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#finalize--";
 title="class or interface in java.lang">finalize, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#getClass--";
 title="class or interface in java.lang">getClass, https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#hashCode--";
 title="class or interfa



[17/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ReplayBatchOperation.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ReplayBatchOperation.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ReplayBatchOperation.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ReplayBatchOperation.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.ReplayBatchOperation.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */




[24/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  pub



[37/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/org/apache/hadoop/hbase/regionserver/Segment.html
--
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/Segment.html 
b/devapidocs/org/apache/hadoop/hbase/regionserver/Segment.html
index 6acb62e..8d6eb3c 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/Segment.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/Segment.html
@@ -18,7 +18,7 @@
 catch(err) {
 }
 //-->
-var methods = 
{"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":9,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":6,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10};
+var methods = 
{"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":9,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":6,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10};
 var tabs = {65535:["t0","All Methods"],1:["t1","Static 
Methods"],2:["t2","Instance Methods"],4:["t3","Abstract 
Methods"],8:["t4","Concrete Methods"]};
 var altColor = "altColor";
 var rowColor = "rowColor";
@@ -108,6 +108,10 @@ var activeTableTab = "activeTableTab";
 
 
 
+All Implemented Interfaces:
+MemStoreSizing
+
+
 Direct Known Subclasses:
 ImmutableSegment, MutableSegment
 
@@ -115,7 +119,8 @@ var activeTableTab = "activeTableTab";
 
 @InterfaceAudience.Private
 public abstract class Segment
-extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true";
 title="class or interface in java.lang">Object
+implements MemStoreSizing
 This is an abstraction of a segment maintained in a 
memstore, e.g., the active
  cell set or its snapshot.
 
@@ -160,12 +165,12 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 memStoreLAB 
 
 
-protected long
-minSequenceId 
+protected MemStoreSizing
+memStoreSizing 
 
 
-protected MemStoreSizing
-segmentSize 
+protected long
+minSequenceId 
 
 
 protected boolean
@@ -176,6 +181,13 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 timeRangeTracker 
 
 
+
+
+
+
+Fields inherited from 
interface org.apache.hadoop.hbase.regionserver.MemStoreSizing
+DUD
+
 
 
 
@@ -274,95 +286,97 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 
 
 
+long
+getDataSize() 
+
+
+long
+getHeapSize() 
+
+
 (package private) MemStoreLAB
 getMemStoreLAB() 
 
-
+
 MemStoreSize
 getMemStoreSize() 
 
-
+
 long
 getMinSequenceId() 
 
-
+
+long
+getOffHeapSize() 
+
+
 protected KeyValueScanner
 getScanner(long readPoint)
 Creates the scanner for the given read point
 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true";
 title="class or interface in java.util">List
 getScanners(long readPoint) 
 
-
+
 TimeRangeTracker
 getTimeRangeTracker() 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/SortedSet.html?is-external=true";
 title="class or interface in java.util">SortedSet
 headSet(Cell firstKeyOnRow) 
 
-
-long
-heapSize() 
-
-
+
 protected long
 heapSizeChange(Cell cell,
   boolean succ) 
 
-
+
+long
+incMemStoreSize(long delta,
+   long heapOverhead,
+   long offHeapOverhead) 
+
+
 void
 incScannerCount() 
 
-
-protected void
-incSize(long delta,
-   long heapOverhead,
-   long offHeapOverhead)
-Updates the size counters of the segment by the given 
delta
-
-
-
+
 protected long
 indexEntryOffHeapSize(boolean offHeap) 
 
-
+
 protected long
 indexEntryOnHeapSize(boolean onHeap) 
 
-
+
 protected abstract long
 indexEntrySize() 
 
-
+
 protected void
 internalAdd(Cell cell,
boolean mslabUsed,
-   MemStoreSizing memstoreSizing) 
+   MemStoreSizing memstoreSizing) 
 
-
+
 boolean
 isEmpty() 
 
-
+
 boolean
 isTagsPresent() 
 
-
+
 https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true";
 title="class or interface in java.util">Iterator
 iterator() 
 
-
-long
-keySize() 
-
-
+
 Cell
 last() 
 
-
+
 Cell
 maybeCloneWithAllocator(Cell cell,
boolean forceCloneOfBigCell)
@@ -372,10 +386,6 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
  When a cell's size is too big (bigger than maxAlloc), it is not allocated on 
MSLAB.
 
 
-
-long
-offHeapSize() 
-
 
 protected long
 offHeapSizeChange(Cell cell,
@@ -409,13 +419,13 @@ extends https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 updateMetaInfo(Cell cellToAdd,




[11/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushHandler.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushHandler.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushHandler.html
index 4c42811..0bc3ddb 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushHandler.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.FlushHandler.html
@@ -563,381 +563,390 @@
 555// If this is first time we've 
been put off, then emit a log message.
 556if (fqe.getRequeueCount() <= 
0) {
 557  // Note: We don't impose 
blockingStoreFiles constraint on meta regions
-558  LOG.warn("Region " + 
region.getRegionInfo().getEncodedName() + " has too many " +
-559"store files; delaying flush 
up to " + this.blockingWaitTime + "ms");
-560  if 
(!this.server.compactSplitThread.requestSplit(region)) {
-561try {
-562  
this.server.compactSplitThread.requestSystemCompaction(region,
-563
Thread.currentThread().getName());
-564} catch (IOException e) {
-565  e = e instanceof 
RemoteException ?
-566  
((RemoteException)e).unwrapRemoteException() : e;
-567  LOG.error("Cache flush 
failed for region " +
-568
Bytes.toStringBinary(region.getRegionInfo().getRegionName()), e);
-569}
-570  }
-571}
-572
-573// Put back on the queue.  Have 
it come back out of the queue
-574// after a delay of 
this.blockingWaitTime / 100 ms.
-575
this.flushQueue.add(fqe.requeue(this.blockingWaitTime / 100));
-576// Tell a lie, it's not flushed 
but it's ok
-577return true;
-578  }
-579}
-580return flushRegion(region, false, 
fqe.isForceFlushAllStores(), fqe.getTracker());
-581  }
-582
-583  /**
-584   * Flush a region.
-585   * @param region Region to flush.
-586   * @param emergencyFlush Set if we are 
being force flushed. If true the region
-587   * needs to be removed from the flush 
queue. If false, when we were called
-588   * from the main flusher run loop and 
we got the entry to flush by calling
-589   * poll on the flush queue (which 
removed it).
-590   * @param forceFlushAllStores whether 
we want to flush all store.
-591   * @return true if the region was 
successfully flushed, false otherwise. If
-592   * false, there will be accompanying 
log messages explaining why the region was
-593   * not flushed.
-594   */
-595  private boolean flushRegion(HRegion 
region, boolean emergencyFlush, boolean forceFlushAllStores,
-596  FlushLifeCycleTracker tracker) {
-597synchronized (this.regionsInQueue) 
{
-598  FlushRegionEntry fqe = 
this.regionsInQueue.remove(region);
-599  // Use the start time of the 
FlushRegionEntry if available
-600  if (fqe != null && 
emergencyFlush) {
-601// Need to remove from region 
from delay queue. When NOT an
-602// emergencyFlush, then item was 
removed via a flushQueue.poll.
-603flushQueue.remove(fqe);
-604  }
-605}
-606
-607tracker.beforeExecution();
-608lock.readLock().lock();
-609try {
-610  notifyFlushRequest(region, 
emergencyFlush);
-611  FlushResult flushResult = 
region.flushcache(forceFlushAllStores, false, tracker);
-612  boolean shouldCompact = 
flushResult.isCompactionNeeded();
-613  // We just want to check the size
-614  boolean shouldSplit = 
region.checkSplit() != null;
-615  if (shouldSplit) {
-616
this.server.compactSplitThread.requestSplit(region);
-617  } else if (shouldCompact) {
-618
server.compactSplitThread.requestSystemCompaction(region, 
Thread.currentThread().getName());
-619  }
-620} catch (DroppedSnapshotException ex) 
{
-621  // Cache flush can fail in a few 
places. If it fails in a critical
-622  // section, we get a 
DroppedSnapshotException and a replay of wal
-623  // is required. Currently the only 
way to do this is a restart of
-624  // the server. Abort because hdfs 
is probably bad (HBASE-644 is a case
-625  // where hdfs was bad but passed 
the hdfs check).
-626  server.abort("Replay of WAL 
required. Forcing server shutdown", ex);
-627  return false;
-628} catch (IOException ex) {
-629  ex = ex instanceof RemoteException 
? ((RemoteException) ex).unwrapRemoteException() : ex;
-630  LOG.error(
-631"Cache flush failed"
-632+ (region != null ? (" for 
region " +
-633
Bytes.toStringBinary(region.getRegionInfo().getRegionName()))
-634  : ""), ex);
-635  if (!server.checkFileSystem()) {
-636return false;
-637  }
-638} finall



[15/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
index 4a879bb..7d27402 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.RowLockImpl.html
@@ -300,7 +300,7 @@
 292  private Map coprocessorServiceHandlers = 
Maps.newHashMap();
 293
 294  // Track data size in all memstores
-295  private final MemStoreSizing 
memStoreSize = new MemStoreSizing();
+295  private final MemStoreSizing 
memStoreSizing = new ThreadSafeMemStoreSizing();
 296  private final RegionServicesForStores 
regionServicesForStores = new RegionServicesForStores(this);
 297
 298  // Debug possible data loss due to WAL 
off
@@ -1218,7389 +1218,7399 @@
 1210   * Increase the size of mem store in 
this region and the size of global mem
 1211   * store
 1212   */
-1213  public void 
incMemStoreSize(MemStoreSize memStoreSize) {
-1214if (this.rsAccounting != null) {
-1215  
rsAccounting.incGlobalMemStoreSize(memStoreSize);
-1216}
-1217long dataSize;
-1218synchronized (this.memStoreSize) {
-1219  
this.memStoreSize.incMemStoreSize(memStoreSize);
-1220  dataSize = 
this.memStoreSize.getDataSize();
-1221}
-1222
checkNegativeMemStoreDataSize(dataSize, memStoreSize.getDataSize());
-1223  }
-1224
-1225  public void 
decrMemStoreSize(MemStoreSize memStoreSize) {
-1226if (this.rsAccounting != null) {
-1227  
rsAccounting.decGlobalMemStoreSize(memStoreSize);
-1228}
-1229long size;
-1230synchronized (this.memStoreSize) {
-1231  
this.memStoreSize.decMemStoreSize(memStoreSize);
-1232  size = 
this.memStoreSize.getDataSize();
+1213  void incMemStoreSize(MemStoreSize mss) 
{
+1214incMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1215  }
+1216
+1217  void incMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1218if (this.rsAccounting != null) {
+1219  
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1220}
+1221long dataSize =
+1222
this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
+1223
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
+1224  }
+1225
+1226  void decrMemStoreSize(MemStoreSize 
mss) {
+1227decrMemStoreSize(mss.getDataSize(), 
mss.getHeapSize(), mss.getOffHeapSize());
+1228  }
+1229
+1230  void decrMemStoreSize(long 
dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+1231if (this.rsAccounting != null) {
+1232  
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, 
offHeapSizeDelta);
 1233}
-1234checkNegativeMemStoreDataSize(size, 
-memStoreSize.getDataSize());
-1235  }
-1236
-1237  private void 
checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {
-1238// This is extremely bad if we make 
memStoreSize negative. Log as much info on the offending
-1239// caller as possible. (memStoreSize 
might be a negative value already -- freeing memory)
-1240if (memStoreDataSize < 0) {
-1241  LOG.error("Asked to modify this 
region's (" + this.toString()
-1242  + ") memStoreSize to a 
negative value which is incorrect. Current memStoreSize="
-1243  + (memStoreDataSize - delta) + 
", delta=" + delta, new Exception());
-1244}
-1245  }
-1246
-1247  @Override
-1248  public RegionInfo getRegionInfo() {
-1249return this.fs.getRegionInfo();
-1250  }
-1251
-1252  /**
-1253   * @return Instance of {@link 
RegionServerServices} used by this HRegion.
-1254   * Can be null.
-1255   */
-1256  RegionServerServices 
getRegionServerServices() {
-1257return this.rsServices;
-1258  }
-1259
-1260  @Override
-1261  public long getReadRequestsCount() {
-1262return readRequestsCount.sum();
-1263  }
-1264
-1265  @Override
-1266  public long 
getFilteredReadRequestsCount() {
-1267return 
filteredReadRequestsCount.sum();
-1268  }
-1269
-1270  @Override
-1271  public long getWriteRequestsCount() 
{
-1272return writeRequestsCount.sum();
-1273  }
-1274
-1275  @Override
-1276  public long getMemStoreDataSize() {
-1277return memStoreSize.getDataSize();
-1278  }
-1279
-1280  @Override
-1281  public long getMemStoreHeapSize() {
-1282return memStoreSize.getHeapSize();
-1283  }
-1284
-1285  @Override
-1286  public long getMemStoreOffHeapSize() 
{
-1287return 
memStoreSize.getOffHeapSize();
-1288  }
-1289
-1290  /** @return store services for this 
region, to access services required by store level needs */
-1291  public RegionServicesForStores 
getReg



[33/51] [partial] hbase-site git commit: Published site at 021f66d11d2cbb7308308093e29e69d6e7661ee9.


2018-05-12

Thread
git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/92a26cfb/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
--
diff --git 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
index 68e703a..d1ce4f8 100644
--- 
a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
+++ 
b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/AbstractMemStore.html
@@ -173,183 +173,177 @@
 165
 166  @Override
 167  public MemStoreSize getSnapshotSize() 
{
-168return getSnapshotSizing();
+168return 
this.snapshot.getMemStoreSize();
 169  }
 170
-171  MemStoreSizing getSnapshotSizing() {
-172return new 
MemStoreSizing(this.snapshot.keySize(),
-173this.snapshot.heapSize(),
-174this.snapshot.offHeapSize());
-175  }
-176
-177  @Override
-178  public String toString() {
-179StringBuilder buf = new 
StringBuilder();
-180int i = 1;
-181try {
-182  for (Segment segment : 
getSegments()) {
-183buf.append("Segment (" + i + ") " 
+ segment.toString() + "; ");
-184i++;
-185  }
-186} catch (IOException e){
-187  return e.toString();
-188}
-189return buf.toString();
-190  }
-191
-192  protected Configuration 
getConfiguration() {
-193return conf;
-194  }
+171  @Override
+172  public String toString() {
+173StringBuilder buf = new 
StringBuilder();
+174int i = 1;
+175try {
+176  for (Segment segment : 
getSegments()) {
+177buf.append("Segment (" + i + ") " 
+ segment.toString() + "; ");
+178i++;
+179  }
+180} catch (IOException e){
+181  return e.toString();
+182}
+183return buf.toString();
+184  }
+185
+186  protected Configuration 
getConfiguration() {
+187return conf;
+188  }
+189
+190  protected void dump(Logger log) {
+191active.dump(log);
+192snapshot.dump(log);
+193  }
+194
 195
-196  protected void dump(Logger log) {
-197active.dump(log);
-198snapshot.dump(log);
-199  }
-200
-201
-202  /*
-203   * Inserts the specified Cell into 
MemStore and deletes any existing
-204   * versions of the same 
row/family/qualifier as the specified Cell.
-205   * 

-206   * First, the specified Cell is 
inserted into the Memstore.
-207   * 

-208   * If there are any existing Cell in 
this MemStore with the same row,
-209   * family, and qualifier, they are 
removed.
-210   * 

-211   * Callers must hold the read lock.
-212   *
-213   * @param cell the cell to be updated
-214   * @param readpoint readpoint below 
which we can safely remove duplicate KVs
-215   * @param memstoreSize
-216   */
-217  private void upsert(Cell cell, long 
readpoint, MemStoreSizing memstoreSizing) {
-218// Add the Cell to the MemStore
-219// Use the internalAdd method here 
since we (a) already have a lock
-220// and (b) cannot safely use the 
MSLAB here without potentially
-221// hitting OOME - see 
TestMemStore.testUpsertMSLAB for a
-222// test that triggers the 
pathological case if we don't avoid MSLAB
-223// here.
-224// This cell data is backed by the 
same byte[] where we read request in RPC(See HBASE-15180). We
-225// must do below deep copy. Or else 
we will keep referring to the bigger chunk of memory and
-226// prevent it from getting GCed.
-227cell = deepCopyIfNeeded(cell);
-228this.active.upsert(cell, readpoint, 
memstoreSizing);
-229setOldestEditTimeToNow();
-230checkActiveSize();
-231  }
-232
-233  /*
-234   * @param a
-235   * @param b
-236   * @return Return lowest of a or b or 
null if both a and b are null
-237   */
-238  protected Cell getLowest(final Cell a, 
final Cell b) {
-239if (a == null) {
-240  return b;
-241}
-242if (b == null) {
-243  return a;
-244}
-245return comparator.compareRows(a, b) 
<= 0? a: b;
-246  }
-247
-248  /*
-249   * @param key Find row that follows 
this one.  If null, return first.
-250   * @param set Set to look in for a row 
beyond row.
-251   * @return Next row or null if none 
found.  If one found, will be a new
-252   * KeyValue -- can be destroyed by 
subsequent calls to this method.
-253   */
-254  protected Cell getNextRow(final Cell 
key,
-255  final NavigableSet set) 
{
-256Cell result = null;
-257SortedSet tail = key == 
null? set: set.tailSet(key);
-258// Iterate until we fall into the 
next row; i.e. move off current row
-259for (Cell cell: tail) {
-260  if (comparator.compareRows(cell, 
key) <= 0) {
-261continue;
-262  }
-263  // Note: Not suppressing deletes or 
expired cells.  Needs to be handled
-264  // by higher up functions.
-265  result = cell;
-266  break;
-267}
-268return result;
-269  }
-270
-271  /**
-272   * If the segment has a memory 
allo


    

  
  

    

      
    
    
86 matches
    

    
Advanced search


Search the list




    
    

      

                               Site Navigation
      
      

        
The Mail Archive home
        

          commits - all messages
        

          commits  - about the list
        
Expand
      
    

    

      

  				Mail list logo
      
      
    
  
  

    

	        	      Footer information
    
    

      
The Mail Archive home
      
Add your mailing list
      
FAQ
      
Support
      
Privacy