[02/50] [abbrv] hbase git commit: HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster restart
HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster
restart
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b336da92
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b336da92
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b336da92
Branch: refs/heads/HBASE-19064
Commit: b336da925ac5c5ee3565112de4b808fe2eed08a2
Parents: 78da0e3
Author: Allan Yang
Authored: Tue Jun 19 09:45:47 2018 +0800
Committer: Allan Yang
Committed: Tue Jun 19 09:45:47 2018 +0800
--
.../src/main/protobuf/HBase.proto | 15 ++
.../org/apache/hadoop/hbase/master/HMaster.java | 8 +
.../hadoop/hbase/master/ServerManager.java | 213 ++-
.../apache/hadoop/hbase/master/TestMaster.java | 33 +++
4 files changed, 268 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-protocol-shaded/src/main/protobuf/HBase.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
index 29067f1..0af2ffd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
@@ -252,4 +252,19 @@ message CacheEvictionStats {
optional int64 bytes_evicted = 2;
optional int64 max_cache_size = 3;
repeated RegionExceptionMessage exception = 4;
+}
+
+message FlushedStoreSequenceId {
+ required bytes family = 1;
+ required uint64 seqId = 2;
+}
+
+message FlushedRegionSequenceId {
+ required bytes regionEncodedName = 1;
+ required uint64 seqId = 2;
+ repeated FlushedStoreSequenceId stores = 3;
+}
+
+message FlushedSequenceId {
+ repeated FlushedRegionSequenceId regionSequenceId = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 883bb4f..38aac50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -862,6 +862,13 @@ public class HMaster extends HRegionServer implements
MasterServices {
status.setStatus("Initializing ZK system trackers");
initializeZKBasedSystemTrackers();
+status.setStatus("Loading last flushed sequence id of regions");
+try {
+ this.serverManager.loadLastFlushedSequenceIds();
+} catch (IOException e) {
+ LOG.debug("Failed to load last flushed sequence id of regions"
+ + " from file system", e);
+}
// Set ourselves as active Master now our claim has succeeded up in zk.
this.activeMaster = true;
@@ -946,6 +953,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
getChoreService().scheduleChore(normalizerChore);
this.catalogJanitorChore = new CatalogJanitor(this);
getChoreService().scheduleChore(catalogJanitorChore);
+this.serverManager.startChore();
status.setStatus("Starting cluster schema service");
initClusterSchemaService();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index c746502..cfbd52f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -38,10 +38,15 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.RegionMetrics;
+import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
@@ -51,9 +56,11 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import
hbase git commit: HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster restart
Repository: hbase
Updated Branches:
refs/heads/master 78da0e366 -> b336da925
HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster
restart
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b336da92
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b336da92
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b336da92
Branch: refs/heads/master
Commit: b336da925ac5c5ee3565112de4b808fe2eed08a2
Parents: 78da0e3
Author: Allan Yang
Authored: Tue Jun 19 09:45:47 2018 +0800
Committer: Allan Yang
Committed: Tue Jun 19 09:45:47 2018 +0800
--
.../src/main/protobuf/HBase.proto | 15 ++
.../org/apache/hadoop/hbase/master/HMaster.java | 8 +
.../hadoop/hbase/master/ServerManager.java | 213 ++-
.../apache/hadoop/hbase/master/TestMaster.java | 33 +++
4 files changed, 268 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-protocol-shaded/src/main/protobuf/HBase.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
index 29067f1..0af2ffd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
@@ -252,4 +252,19 @@ message CacheEvictionStats {
optional int64 bytes_evicted = 2;
optional int64 max_cache_size = 3;
repeated RegionExceptionMessage exception = 4;
+}
+
+message FlushedStoreSequenceId {
+ required bytes family = 1;
+ required uint64 seqId = 2;
+}
+
+message FlushedRegionSequenceId {
+ required bytes regionEncodedName = 1;
+ required uint64 seqId = 2;
+ repeated FlushedStoreSequenceId stores = 3;
+}
+
+message FlushedSequenceId {
+ repeated FlushedRegionSequenceId regionSequenceId = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 883bb4f..38aac50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -862,6 +862,13 @@ public class HMaster extends HRegionServer implements
MasterServices {
status.setStatus("Initializing ZK system trackers");
initializeZKBasedSystemTrackers();
+status.setStatus("Loading last flushed sequence id of regions");
+try {
+ this.serverManager.loadLastFlushedSequenceIds();
+} catch (IOException e) {
+ LOG.debug("Failed to load last flushed sequence id of regions"
+ + " from file system", e);
+}
// Set ourselves as active Master now our claim has succeeded up in zk.
this.activeMaster = true;
@@ -946,6 +953,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
getChoreService().scheduleChore(normalizerChore);
this.catalogJanitorChore = new CatalogJanitor(this);
getChoreService().scheduleChore(catalogJanitorChore);
+this.serverManager.startChore();
status.setStatus("Starting cluster schema service");
initClusterSchemaService();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index c746502..cfbd52f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -38,10 +38,15 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.RegionMetrics;
+import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
@@ -51,9 +56,11 @@ import
