[hbase] branch master updated: HBASE-22870 reflection fails to access a private nested class
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 7697d48 HBASE-22870 reflection fails to access a private nested class
7697d48 is described below
commit 7697d48cd7b636396d5f73be43326a4501a9ea43
Author: satanson
AuthorDate: Sun Aug 18 09:47:06 2019 +0800
HBASE-22870 reflection fails to access a private nested class
Signed-off-by Reid Chan
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java| 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 61361be..a0a6b4c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2475,9 +2475,11 @@ public class HRegionServer extends HasThread implements
this.abortMonitor = new Timer("Abort regionserver monitor", true);
TimerTask abortTimeoutTask = null;
try {
-abortTimeoutTask =
+Constructor timerTaskCtor =
Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
-
.asSubclass(TimerTask.class).getDeclaredConstructor().newInstance();
+.asSubclass(TimerTask.class).getDeclaredConstructor();
+timerTaskCtor.setAccessible(true);
+abortTimeoutTask = timerTaskCtor.newInstance();
} catch (Exception e) {
LOG.warn("Initialize abort timeout task failed", e);
}
[hbase] branch branch-2 updated: HBASE-22870 reflection fails to access a private nested class
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 29ed415 HBASE-22870 reflection fails to access a private nested class
29ed415 is described below
commit 29ed4157b30d9720337d576ad48c28e9d1c47114
Author: satanson
AuthorDate: Sun Aug 18 09:47:06 2019 +0800
HBASE-22870 reflection fails to access a private nested class
Signed-off-by Reid Chan
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java| 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c4fd11c..604cf11 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -2480,9 +2480,11 @@ public class HRegionServer extends HasThread implements
this.abortMonitor = new Timer("Abort regionserver monitor", true);
TimerTask abortTimeoutTask = null;
try {
-abortTimeoutTask =
+Constructor timerTaskCtor =
Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
-
.asSubclass(TimerTask.class).getDeclaredConstructor().newInstance();
+.asSubclass(TimerTask.class).getDeclaredConstructor();
+timerTaskCtor.setAccessible(true);
+abortTimeoutTask = timerTaskCtor.newInstance();
} catch (Exception e) {
LOG.warn("Initialize abort timeout task failed", e);
}
[hbase] branch branch-2.2 updated: HBASE-22870 reflection fails to access a private nested class
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 754c2da HBASE-22870 reflection fails to access a private nested class
754c2da is described below
commit 754c2da349c7c0aea40f024f52c1f9191bf3cad2
Author: satanson
AuthorDate: Sun Aug 18 09:47:06 2019 +0800
HBASE-22870 reflection fails to access a private nested class
Signed-off-by Reid Chan
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7991a61..81166e2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1065,9 +1065,11 @@ public class HRegionServer extends HasThread implements
Timer abortMonitor = new Timer("Abort regionserver monitor", true);
TimerTask abortTimeoutTask = null;
try {
-abortTimeoutTask =
-Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
-
.asSubclass(TimerTask.class).getDeclaredConstructor().newInstance();
+Constructor timerTaskCtor =
+ Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
+.asSubclass(TimerTask.class).getDeclaredConstructor();
+timerTaskCtor.setAccessible(true);
+abortTimeoutTask = timerTaskCtor.newInstance();
} catch (Exception e) {
LOG.warn("Initialize abort timeout task failed", e);
}
[hbase] branch branch-2.1 updated: HBASE-22870 reflection fails to access a private nested class
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9e18049 HBASE-22870 reflection fails to access a private nested class
9e18049 is described below
commit 9e180499577bac0a103eb965055d961d19e15acb
Author: satanson
AuthorDate: Sun Aug 18 09:47:06 2019 +0800
HBASE-22870 reflection fails to access a private nested class
Signed-off-by Reid Chan
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
---
.../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 405d75a..587e7f6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1047,9 +1047,11 @@ public class HRegionServer extends HasThread implements
Timer abortMonitor = new Timer("Abort regionserver monitor", true);
TimerTask abortTimeoutTask = null;
try {
-abortTimeoutTask =
-Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
-
.asSubclass(TimerTask.class).getDeclaredConstructor().newInstance();
+Constructor timerTaskCtor =
+ Class.forName(conf.get(ABORT_TIMEOUT_TASK,
SystemExitWhenAbortTimeout.class.getName()))
+.asSubclass(TimerTask.class).getDeclaredConstructor();
+timerTaskCtor.setAccessible(true);
+abortTimeoutTask = timerTaskCtor.newInstance();
} catch (Exception e) {
LOG.warn("Initialize abort timeout task failed", e);
}
[hbase] branch branch-1 updated: HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 105008e HBASE-22861 [WAL] Merged region should get its WAL according
to WALProvider
105008e is described below
commit 105008e748da81f11d3e111932ce24ee03bca64a
Author: Reid Chan
AuthorDate: Thu Aug 22 11:15:21 2019 +0800
HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/MockRegionServerServicesWithWALs.java| 296 +
.../regionserver/TestRegionMergeTransaction.java | 40 ++-
.../hbase/regionserver/TestSplitTransaction.java | 262 +-
4 files changed, 337 insertions(+), 264 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index b137c97..1157eba 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7198,7 +7198,8 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
*/
HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
final HRegion region_b) throws IOException {
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
+WAL mergedRegionWAL = rsServices == null ? getWAL() :
rsServices.getWAL(mergedRegionInfo);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), mergedRegionWAL,
fs.getFileSystem(), this.getBaseConf(), mergedRegionInfo,
this.getTableDesc(), this.rsServices);
r.readRequestsCount.set(this.getReadRequestsCount()
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
new file mode 100644
index 000..69290b2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import com.google.protobuf.Service;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
+import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager;
+import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
+import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALProvider;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Mock region server services with WALProvider, it can be used for testing
wal related tests,
+ * like split or merge regions.
+ */
+public class MockRegionServerService
[hbase] branch branch-1.4 updated: HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new dde3873 HBASE-22861 [WAL] Merged region should get its WAL according
to WALProvider
dde3873 is described below
commit dde3873360bc64d81d1dbdfa5f79bf2096954fbc
Author: Reid Chan
AuthorDate: Thu Aug 22 11:15:21 2019 +0800
HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/MockRegionServerServicesWithWALs.java| 296 +
.../regionserver/TestRegionMergeTransaction.java | 40 ++-
.../hbase/regionserver/TestSplitTransaction.java | 262 +-
4 files changed, 337 insertions(+), 264 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9ccb677..f16e9e9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7187,7 +7187,8 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
*/
HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
final HRegion region_b) throws IOException {
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
+WAL mergedRegionWAL = rsServices == null ? getWAL() :
rsServices.getWAL(mergedRegionInfo);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), mergedRegionWAL,
fs.getFileSystem(), this.getBaseConf(), mergedRegionInfo,
this.getTableDesc(), this.rsServices);
r.readRequestsCount.set(this.getReadRequestsCount()
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
new file mode 100644
index 000..69290b2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import com.google.protobuf.Service;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
+import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager;
+import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
+import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALProvider;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Mock region server services with WALProvider, it can be used for testing
wal related tests,
+ * like split or merge regions.
+ */
+public class MockRegionServerService
[hbase] branch branch-1.3 updated: HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new ca0390c HBASE-22861 [WAL] Merged region should get its WAL according
to WALProvider
ca0390c is described below
commit ca0390cb7b17dc457a52fc210f592ecf0591fbfa
Author: Reid Chan
AuthorDate: Thu Aug 22 11:15:21 2019 +0800
HBASE-22861 [WAL] Merged region should get its WAL according to WALProvider
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/MockRegionServerServicesWithWALs.java| 296 +
.../regionserver/TestRegionMergeTransaction.java | 40 ++-
.../hbase/regionserver/TestSplitTransaction.java | 258 +-
4 files changed, 337 insertions(+), 260 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c988863..5a930d6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7014,7 +7014,8 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
*/
HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
final HRegion region_b) throws IOException {
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
+WAL mergedRegionWAL = rsServices == null ? getWAL() :
rsServices.getWAL(mergedRegionInfo);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), mergedRegionWAL,
fs.getFileSystem(), this.getBaseConf(), mergedRegionInfo,
this.getTableDesc(), this.rsServices);
r.readRequestsCount.set(this.getReadRequestsCount()
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
new file mode 100644
index 000..69290b2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import com.google.protobuf.Service;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
+import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager;
+import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.MetricsRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
+import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALProvider;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Mock region server services with WALProvider
[hbase] branch branch-1.3 updated: HBASE-22861 [Addendum] Remove unassign(not exist in branch-1.3) method in MockRegionServerServicesWithWALs
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new a4a7c0e HBASE-22861 [Addendum] Remove unassign(not exist in
branch-1.3) method in MockRegionServerServicesWithWALs
a4a7c0e is described below
commit a4a7c0edc259ed49760f752881bd8a81b8255452
Author: Reid Chan
AuthorDate: Thu Aug 22 12:36:00 2019 +0800
HBASE-22861 [Addendum] Remove unassign(not exist in branch-1.3) method in
MockRegionServerServicesWithWALs
---
.../org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java| 5 -
1 file changed, 5 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
index 69290b2..7f2606f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServicesWithWALs.java
@@ -199,11 +199,6 @@ public class MockRegionServerServicesWithWALs implements
RegionServerServices {
}
@Override
- public void unassign(byte[] regionName) throws IOException {
-rss.unassign(regionName);
- }
-
- @Override
public void addToOnlineRegions(Region r) {
rss.addToOnlineRegions(r);
}
[hbase] branch branch-1.3 updated: HBASE-22835 Scan/Get with setColumn and the store with ROWCOL bloom filter could throw AssertionError
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 957d968 HBASE-22835 Scan/Get with setColumn and the store with ROWCOL
bloom filter could throw AssertionError
957d968 is described below
commit 957d968cca24efacae77b65c1761bac93e7b4925
Author: eomiks
AuthorDate: Fri Aug 23 11:57:59 2019 +0900
HBASE-22835 Scan/Get with setColumn and the store with ROWCOL bloom filter
could throw AssertionError
Signed-off-by Reid Chan
---
.../hadoop/hbase/regionserver/StoreScanner.java| 227 -
.../apache/hadoop/hbase/HBaseTestingUtility.java | 29 +++
.../hbase/regionserver/TestIsDeleteFailure.java| 146 +
3 files changed, 305 insertions(+), 97 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 9d63374..3da0d1c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.CountDownLatch;
@@ -42,7 +41,6 @@ import org.apache.hadoop.hbase.client.IsolationLevel;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState;
import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler;
@@ -552,8 +550,7 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
prevCell = cell;
topChanged = false;
ScanQueryMatcher.MatchCode qcode = matcher.match(cell);
- qcode = optimize(qcode, cell);
- switch(qcode) {
+ switch (qcode) {
case INCLUDE:
case INCLUDE_AND_SEEK_NEXT_ROW:
case INCLUDE_AND_SEEK_NEXT_COL:
@@ -606,9 +603,9 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
// the heap.peek() will any way be in the next row. So the
SQM.match(cell) need do
// another compareRow to say the current row is DONE
matcher.row = null;
-seekToNextRow(cell);
+seekOrSkipToNextRow(cell);
} else if (qcode ==
ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
-seekAsDirection(matcher.getKeyForNextColumn(cell));
+seekOrSkipToNextColumn(cell);
} else {
this.heap.next();
}
@@ -648,7 +645,7 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
// the heap.peek() will any way be in the next row. So the
SQM.match(cell) need do
// another compareRow to say the current row is DONE
matcher.row = null;
- seekToNextRow(cell);
+ seekOrSkipToNextRow(cell);
NextState stateAfterSeekNextRow = needToReturn(outResult);
if (stateAfterSeekNextRow != null) {
return
scannerContext.setScannerState(stateAfterSeekNextRow).hasMoreValues();
@@ -656,7 +653,7 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
break;
case SEEK_NEXT_COL:
- seekAsDirection(matcher.getKeyForNextColumn(cell));
+ seekOrSkipToNextColumn(cell);
NextState stateAfterSeekNextColumn = needToReturn(outResult);
if (stateAfterSeekNextColumn != null) {
return
scannerContext.setScannerState(stateAfterSeekNextColumn).hasMoreValues();
@@ -713,93 +710,6 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
return null;
}
- /**
- * See if we should actually SEEK or rather just SKIP to the next Cell (see
HBASE-13109).
- * This method works together with ColumnTrackers and Filters.
ColumnTrackers may issue SEEK
- * hints, such as seek to next column, next row, or seek to an arbitrary
seek key.
- * This method intercepts these qcodes and decides whether a seek is the
most efficient _actual_
- * way to get us to the requested cell (SEEKs are more expensive than SKIP,
SKIP, SKIP inside the
- * current, loaded block).
- * It does this by looking at the next indexed key of the current HFile.
This key
- * is then compared with the _SEEK_ key, where a SEEK key is an artificial
'last possible key
- * on the row' (on
[hbase] branch branch-1 updated: HBASE-22880 Move the DirScanPool out and do not use static field
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 8961315 HBASE-22880 Move the DirScanPool out and do not use static
field
8961315 is described below
commit 896131540a72f5997d4b02c449e1fb6a62d687a1
Author: Dean_19 <35092554+zha...@users.noreply.github.com>
AuthorDate: Sat Aug 24 13:13:32 2019 +0800
HBASE-22880 Move the DirScanPool out and do not use static field
Signed-off-by Reid Chan
---
.../org/apache/hadoop/hbase/master/HMaster.java| 31 ++---
.../hadoop/hbase/master/cleaner/CleanerChore.java | 146 -
.../hadoop/hbase/master/cleaner/DirScanPool.java | 111
.../hadoop/hbase/master/cleaner/HFileCleaner.java | 8 +-
.../hadoop/hbase/master/cleaner/LogCleaner.java| 10 +-
.../hadoop/hbase/backup/TestHFileArchiving.java| 33 +++--
.../example/TestZooKeeperTableArchiveClient.java | 17 +--
.../hbase/master/cleaner/TestCleanerChore.java | 49 ---
.../hbase/master/cleaner/TestHFileCleaner.java | 21 +--
.../hbase/master/cleaner/TestHFileLinkCleaner.java | 24 +++-
.../hbase/master/cleaner/TestLogsCleaner.java | 8 +-
11 files changed, 252 insertions(+), 206 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f319ee4..1518f76 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -98,7 +98,7 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
-import org.apache.hadoop.hbase.master.cleaner.CleanerChore;
+import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore;
@@ -333,6 +333,7 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
private SnapshotCleanerChore snapshotCleanerChore = null;
CatalogJanitor catalogJanitorChore;
+ private DirScanPool cleanerPool;
private ReplicationZKLockCleanerChore replicationZKLockCleanerChore;
private ReplicationZKNodeCleanerChore replicationZKNodeCleanerChore;
private LogCleaner logCleaner;
@@ -898,6 +899,7 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
(System.currentTimeMillis() - masterActiveTime) / 1000.0f));
this.masterFinishedInitializationTime = System.currentTimeMillis();
configurationManager.registerObserver(this.balancer);
+configurationManager.registerObserver(this.cleanerPool);
configurationManager.registerObserver(this.hfileCleaner);
configurationManager.registerObserver(this.logCleaner);
@@ -1237,22 +1239,19 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
startProcedureExecutor();
-// Initial cleaner chore
-CleanerChore.initChorePool(conf);
- // Start log cleaner thread
- int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 *
1000);
- this.logCleaner =
- new LogCleaner(cleanerInterval,
- this, conf, getMasterFileSystem().getOldLogDir().getFileSystem(conf),
- getMasterFileSystem().getOldLogDir());
-getChoreService().scheduleChore(logCleaner);
-
+// Create cleaner thread pool
+cleanerPool = new DirScanPool(conf);
+// Start log cleaner thread
+int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 *
1000);
+this.logCleaner = new LogCleaner(cleanerInterval, this, conf,
+ getMasterFileSystem().getOldLogDir().getFileSystem(conf),
+ getMasterFileSystem().getOldLogDir(), cleanerPool);
//start the hfile archive cleaner thread
Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
Map params = new HashMap();
params.put(MASTER, this);
-this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
getMasterFileSystem()
-.getFileSystem(), archiveDir, params);
+this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
+ getMasterFileSystem().getFileSystem(), archiveDir, cleanerPool, params);
getChoreService().scheduleChore(hfileCleaner);
final boolean isSnapshotChoreDisabled =
conf.getBoolean(HConstants.SNAPSHOT_CLEANER_DISABLE,
@@ -1306,8 +1305,10 @@ public class HMaster extends HRegionServer implemen
[hbase] branch branch-1.4 updated: HBASE-22880 Move the DirScanPool out and do not use static field
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new fdccda3 HBASE-22880 Move the DirScanPool out and do not use static
field
fdccda3 is described below
commit fdccda3197bd5c9dcbc38e8b49ae19cc8c5d9b13
Author: Dean_19 <35092554+zha...@users.noreply.github.com>
AuthorDate: Sat Aug 24 13:13:32 2019 +0800
HBASE-22880 Move the DirScanPool out and do not use static field
Signed-off-by Reid Chan
---
.../org/apache/hadoop/hbase/master/HMaster.java| 31 ++---
.../hadoop/hbase/master/cleaner/CleanerChore.java | 146 -
.../hadoop/hbase/master/cleaner/DirScanPool.java | 111
.../hadoop/hbase/master/cleaner/HFileCleaner.java | 8 +-
.../hadoop/hbase/master/cleaner/LogCleaner.java| 10 +-
.../hadoop/hbase/backup/TestHFileArchiving.java| 33 +++--
.../example/TestZooKeeperTableArchiveClient.java | 17 +--
.../hbase/master/cleaner/TestCleanerChore.java | 49 ---
.../hbase/master/cleaner/TestHFileCleaner.java | 21 +--
.../hbase/master/cleaner/TestHFileLinkCleaner.java | 24 +++-
.../hbase/master/cleaner/TestLogsCleaner.java | 8 +-
11 files changed, 252 insertions(+), 206 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0bc6c91..7a49623 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -98,7 +98,7 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
-import org.apache.hadoop.hbase.master.cleaner.CleanerChore;
+import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore;
@@ -330,6 +330,7 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
private PeriodicDoMetrics periodicDoMetricsChore = null;
CatalogJanitor catalogJanitorChore;
+ private DirScanPool cleanerPool;
private ReplicationZKLockCleanerChore replicationZKLockCleanerChore;
private ReplicationZKNodeCleanerChore replicationZKNodeCleanerChore;
private LogCleaner logCleaner;
@@ -895,6 +896,7 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
(System.currentTimeMillis() - masterActiveTime) / 1000.0f));
this.masterFinishedInitializationTime = System.currentTimeMillis();
configurationManager.registerObserver(this.balancer);
+configurationManager.registerObserver(this.cleanerPool);
configurationManager.registerObserver(this.hfileCleaner);
configurationManager.registerObserver(this.logCleaner);
@@ -1234,22 +1236,19 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
startProcedureExecutor();
-// Initial cleaner chore
-CleanerChore.initChorePool(conf);
- // Start log cleaner thread
- int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 *
1000);
- this.logCleaner =
- new LogCleaner(cleanerInterval,
- this, conf, getMasterFileSystem().getOldLogDir().getFileSystem(conf),
- getMasterFileSystem().getOldLogDir());
-getChoreService().scheduleChore(logCleaner);
-
+// Create cleaner thread pool
+cleanerPool = new DirScanPool(conf);
+// Start log cleaner thread
+int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 *
1000);
+this.logCleaner = new LogCleaner(cleanerInterval, this, conf,
+ getMasterFileSystem().getOldLogDir().getFileSystem(conf),
+ getMasterFileSystem().getOldLogDir(), cleanerPool);
//start the hfile archive cleaner thread
Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
Map params = new HashMap();
params.put(MASTER, this);
-this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
getMasterFileSystem()
-.getFileSystem(), archiveDir, params);
+this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf,
+ getMasterFileSystem().getFileSystem(), archiveDir, cleanerPool, params);
getChoreService().scheduleChore(hfileCleaner);
serviceStarted = true;
if (LOG.isTraceEnabled()) {
@@ -1291,8 +1290,10 @@ public class HMaster extends HRegionServer implements
MasterServices, Serve
[hbase] branch branch-1 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new f240ca0 HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
f240ca0 is described below
commit f240ca0e637bbbd8c7854703b57d8ceb95b49166
Author: Aman Poonia
AuthorDate: Sat Aug 17 15:17:42 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 30 --
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index c18a49a..b346c14 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -107,6 +107,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+if (!splitEnabled && !mergeEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
@@ -141,19 +158,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
while (candidateIdx < tableRegions.size()) {
HRegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch branch-1.4 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 23991cb HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
23991cb is described below
commit 23991cb51e5bac3a27237475f00bb4976a83d5fe
Author: Aman Poonia
AuthorDate: Sat Aug 17 15:17:42 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 30 --
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index c18a49a..b346c14 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -107,6 +107,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+if (!splitEnabled && !mergeEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
@@ -141,19 +158,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
while (candidateIdx < tableRegions.size()) {
HRegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch branch-1.3 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 7c43b91 HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
7c43b91 is described below
commit 7c43b91c5da8a955cfebe388ad8ea5ada58b9727
Author: Aman Poonia
AuthorDate: Sat Aug 17 15:17:42 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 30 --
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 67aaee1..52714b2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -107,6 +107,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (ServiceException se) {
+ LOG.debug("Unable to determine whether split is enabled", se);
+}
+if (!splitEnabled && !mergeEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
@@ -137,19 +154,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (ServiceException se) {
- LOG.debug("Unable to determine whether split is enabled", se);
-}
while (candidateIdx < tableRegions.size()) {
HRegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch branch-2 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 1eac16e HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
1eac16e is described below
commit 1eac16e78f37102f2595fd0c36dbf85224109607
Author: Aman Poonia
AuthorDate: Mon Aug 26 17:45:01 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 31 --
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 300c6a7..74b338b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -120,7 +120,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
-
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+if (!mergeEnabled && !splitEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList<>();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);
@@ -178,19 +194,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
while (candidateIdx < tableRegions.size()) {
RegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch branch-2.1 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 06ae766 HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
06ae766 is described below
commit 06ae7668095fde5af9e9e18aa2cb47f77f4583fa
Author: Aman Poonia
AuthorDate: Mon Aug 26 17:45:01 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 31 --
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 300c6a7..74b338b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -120,7 +120,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
-
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+if (!mergeEnabled && !splitEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList<>();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);
@@ -178,19 +194,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
while (candidateIdx < tableRegions.size()) {
RegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch branch-2.2 updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new d48225c HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
d48225c is described below
commit d48225c8bfeb4a2ba8d3c5c3a6fa2e8c6592e564
Author: Aman Poonia
AuthorDate: Mon Aug 26 17:45:01 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 31 --
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 300c6a7..74b338b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -120,7 +120,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
-
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+if (!mergeEnabled && !splitEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList<>();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);
@@ -178,19 +194,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
while (candidateIdx < tableRegions.size()) {
RegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch master updated: HBASE-22872 Don't try to create normalization plan unnecesarily when split and merge both are disabled
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 64f8890 HBASE-22872 Don't try to create normalization plan
unnecesarily when split and merge both are disabled
64f8890 is described below
commit 64f88906f7cc7265fe0c42a4c42530dbd660c70b
Author: Aman Poonia
AuthorDate: Tue Aug 27 21:35:57 2019 +0530
HBASE-22872 Don't try to create normalization plan unnecesarily when split
and merge both are disabled
Signed-off-by: Reid Chan
---
.../master/normalizer/SimpleRegionNormalizer.java | 31 --
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index a30a13b..b55d2b6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -131,7 +131,23 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Normalization of system table " + table + " isn't allowed");
return null;
}
-
+boolean splitEnabled = true, mergeEnabled = true;
+try {
+ splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+try {
+ mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
+ LOG.debug("Unable to determine whether split is enabled", e);
+}
+if (!mergeEnabled && !splitEnabled) {
+ LOG.debug("Both split and merge are disabled for table: " + table);
+ return null;
+}
List plans = new ArrayList<>();
List tableRegions =
masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);
@@ -189,19 +205,6 @@ public class SimpleRegionNormalizer implements
RegionNormalizer {
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
int candidateIdx = 0;
-boolean splitEnabled = true, mergeEnabled = true;
-try {
- splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
-try {
- mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
-
RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
-} catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException
e) {
- LOG.debug("Unable to determine whether split is enabled", e);
-}
while (candidateIdx < tableRegions.size()) {
RegionInfo hri = tableRegions.get(candidateIdx);
long regionSize = getRegionSize(hri);
[hbase] branch master updated: HBASE-22928 ScanMetrics counter update may not happen in case of exception in TableRecordReaderImpl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 13b2edc HBASE-22928 ScanMetrics counter update may not happen in case
of exception in TableRecordReaderImpl
13b2edc is described below
commit 13b2edc1be9b389140974ce8d5150aa60407277d
Author: Pankaj
AuthorDate: Mon Sep 2 08:17:44 2019 +0530
HBASE-22928 ScanMetrics counter update may not happen in case of exception
in TableRecordReaderImpl
Signed-off-by: Reid Chan
---
.../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java| 6 ++
1 file changed, 6 insertions(+)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index d7a2ccb..4aac38e 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -77,6 +77,10 @@ public class TableRecordReaderImpl {
* @throws IOException When restarting fails.
*/
public void restart(byte[] firstRow) throws IOException {
+// Update counter metrics based on current scan before reinitializing it
+if (currentScan != null) {
+ updateCounters();
+}
currentScan = new Scan(scan);
currentScan.withStartRow(firstRow);
currentScan.setScanMetricsEnabled(true);
@@ -219,6 +223,7 @@ public class TableRecordReaderImpl {
} catch (IOException e) {
// do not retry if the exception tells us not to do so
if (e instanceof DoNotRetryIOException) {
+ updateCounters();
throw e;
}
// try to handle all other IOExceptions by restarting
@@ -257,6 +262,7 @@ public class TableRecordReaderImpl {
updateCounters();
return false;
} catch (IOException ioe) {
+ updateCounters();
if (logScannerActivity) {
long now = System.currentTimeMillis();
LOG.info("Mapper took " + (now-timestamp)
[hbase] branch branch-2 updated: HBASE-22928 ScanMetrics counter update may not happen in case of exception in TableRecordReaderImpl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 58319b8 HBASE-22928 ScanMetrics counter update may not happen in case
of exception in TableRecordReaderImpl
58319b8 is described below
commit 58319b8cc63495f1130a6d4918429eda21914f20
Author: Pankaj
AuthorDate: Mon Sep 2 08:17:44 2019 +0530
HBASE-22928 ScanMetrics counter update may not happen in case of exception
in TableRecordReaderImpl
Signed-off-by: Reid Chan
---
.../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java| 6 ++
1 file changed, 6 insertions(+)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 28f4da1..1fa943b 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -77,6 +77,10 @@ public class TableRecordReaderImpl {
* @throws IOException When restarting fails.
*/
public void restart(byte[] firstRow) throws IOException {
+// Update counter metrics based on current scan before reinitializing it
+if (currentScan != null) {
+ updateCounters();
+}
currentScan = new Scan(scan);
currentScan.withStartRow(firstRow);
currentScan.setScanMetricsEnabled(true);
@@ -219,6 +223,7 @@ public class TableRecordReaderImpl {
} catch (IOException e) {
// do not retry if the exception tells us not to do so
if (e instanceof DoNotRetryIOException) {
+ updateCounters();
throw e;
}
// try to handle all other IOExceptions by restarting
@@ -257,6 +262,7 @@ public class TableRecordReaderImpl {
updateCounters();
return false;
} catch (IOException ioe) {
+ updateCounters();
if (logScannerActivity) {
long now = System.currentTimeMillis();
LOG.info("Mapper took " + (now-timestamp)
[hbase] branch branch-2.1 updated: HBASE-22928 ScanMetrics counter update may not happen in case of exception in TableRecordReaderImpl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0e87e86 HBASE-22928 ScanMetrics counter update may not happen in case
of exception in TableRecordReaderImpl
0e87e86 is described below
commit 0e87e86a290c20a0a24680ee47512f480b55b6cb
Author: Pankaj
AuthorDate: Mon Sep 2 08:17:44 2019 +0530
HBASE-22928 ScanMetrics counter update may not happen in case of exception
in TableRecordReaderImpl
Signed-off-by: Reid Chan
---
.../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java| 6 ++
1 file changed, 6 insertions(+)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 20c7b94..b9ed629 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -77,6 +77,10 @@ public class TableRecordReaderImpl {
* @throws IOException When restarting fails.
*/
public void restart(byte[] firstRow) throws IOException {
+// Update counter metrics based on current scan before reinitializing it
+if (currentScan != null) {
+ updateCounters();
+}
currentScan = new Scan(scan);
currentScan.withStartRow(firstRow);
currentScan.setScanMetricsEnabled(true);
@@ -219,6 +223,7 @@ public class TableRecordReaderImpl {
} catch (IOException e) {
// do not retry if the exception tells us not to do so
if (e instanceof DoNotRetryIOException) {
+ updateCounters();
throw e;
}
// try to handle all other IOExceptions by restarting
@@ -257,6 +262,7 @@ public class TableRecordReaderImpl {
updateCounters();
return false;
} catch (IOException ioe) {
+ updateCounters();
if (logScannerActivity) {
long now = System.currentTimeMillis();
LOG.info("Mapper took " + (now-timestamp)
[hbase] branch branch-2.2 updated: HBASE-22928 ScanMetrics counter update may not happen in case of exception in TableRecordReaderImpl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new be41a76 HBASE-22928 ScanMetrics counter update may not happen in case
of exception in TableRecordReaderImpl
be41a76 is described below
commit be41a767cf0169117d3530fb3723246a4023da33
Author: Pankaj
AuthorDate: Mon Sep 2 08:17:44 2019 +0530
HBASE-22928 ScanMetrics counter update may not happen in case of exception
in TableRecordReaderImpl
Signed-off-by: Reid Chan
---
.../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java| 6 ++
1 file changed, 6 insertions(+)
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 28f4da1..1fa943b 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -77,6 +77,10 @@ public class TableRecordReaderImpl {
* @throws IOException When restarting fails.
*/
public void restart(byte[] firstRow) throws IOException {
+// Update counter metrics based on current scan before reinitializing it
+if (currentScan != null) {
+ updateCounters();
+}
currentScan = new Scan(scan);
currentScan.withStartRow(firstRow);
currentScan.setScanMetricsEnabled(true);
@@ -219,6 +223,7 @@ public class TableRecordReaderImpl {
} catch (IOException e) {
// do not retry if the exception tells us not to do so
if (e instanceof DoNotRetryIOException) {
+ updateCounters();
throw e;
}
// try to handle all other IOExceptions by restarting
@@ -257,6 +262,7 @@ public class TableRecordReaderImpl {
updateCounters();
return false;
} catch (IOException ioe) {
+ updateCounters();
if (logScannerActivity) {
long now = System.currentTimeMillis();
LOG.info("Mapper took " + (now-timestamp)
[hbase] branch branch-1 updated: HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 5bf60ec HBASE-22890 Verify the files when RegionServer is starting
and BucketCache is in file mode
5bf60ec is described below
commit 5bf60ec55fdf637d80492b61e6e6d8d605c5ef4a
Author: zbq.dean
AuthorDate: Mon Sep 16 15:14:39 2019 +0800
HBASE-22890 Verify the files when RegionServer is starting and BucketCache
is in file mode
Signed-off-by: Reid Chan
Signed-off-by: Stack
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 75 --
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 152 ++-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 59
.../hbase/io/hfile/bucket/TestFileIOEngine.java| 2 +-
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 297 +
5 files changed, 553 insertions(+), 32 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index c5a1b21..98abfc8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -29,6 +29,7 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -69,6 +70,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -242,6 +244,17 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private String ioEngineName;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -252,8 +265,7 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws FileNotFoundException, IOException {
-this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
+ throws IOException {
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -275,6 +287,7 @@ public class BucketCache implements BlockCache, HeapSize {
", memoryFactor: " + memoryFactor);
this.cacheCapacity = capacity;
+this.ioEngineName = ioEngineName;
this.persistencePath = persistencePath;
this.blockSize = blockSize;
this.ioErrorsTolerationDuration = ioErrorsTolerationDuration;
@@ -288,14 +301,15 @@ public class BucketCache implements BlockCache, HeapSize {
this.ramCache = new ConcurrentHashMap();
this.backingMap = new ConcurrentHashMap((int)
blockNumCapacity);
-
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
+ioEngine = getIOEngineFromName();
if (ioEngine.isPersistent() && persistencePath != null) {
try {
retrieveFromFile(bucketSizes);
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
throw new RuntimeException(cnfe);
}
}
@@ -359,12 +373,10 @@ public class BucketCache implements BlockCache, HeapSize {
/**
* G
[hbase] branch branch-1.4 updated: HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new f18b06d HBASE-22890 Verify the files when RegionServer is starting
and BucketCache is in file mode
f18b06d is described below
commit f18b06da1c2a106b16e2f863f01dc5ec9d4389a8
Author: zbq.dean
AuthorDate: Mon Sep 16 15:14:39 2019 +0800
HBASE-22890 Verify the files when RegionServer is starting and BucketCache
is in file mode
Signed-off-by: Reid Chan
Signed-off-by: Stack
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 75 --
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 152 ++-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 59
.../hbase/io/hfile/bucket/TestFileIOEngine.java| 2 +-
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 297 +
5 files changed, 553 insertions(+), 32 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 5a4ac13..af10f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -29,6 +29,7 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -69,6 +70,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -242,6 +244,17 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private String ioEngineName;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -252,8 +265,7 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws FileNotFoundException, IOException {
-this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
+ throws IOException {
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -275,6 +287,7 @@ public class BucketCache implements BlockCache, HeapSize {
", memoryFactor: " + memoryFactor);
this.cacheCapacity = capacity;
+this.ioEngineName = ioEngineName;
this.persistencePath = persistencePath;
this.blockSize = blockSize;
this.ioErrorsTolerationDuration = ioErrorsTolerationDuration;
@@ -288,14 +301,15 @@ public class BucketCache implements BlockCache, HeapSize {
this.ramCache = new ConcurrentHashMap();
this.backingMap = new ConcurrentHashMap((int)
blockNumCapacity);
-
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
+ioEngine = getIOEngineFromName();
if (ioEngine.isPersistent() && persistencePath != null) {
try {
retrieveFromFile(bucketSizes);
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
throw new RuntimeException(cnfe);
}
}
@@ -359,12 +373,10 @@ public class BucketCache implements BlockCache, HeapSize {
[hbase] branch branch-1 updated: Revert "HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode"
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 3b6cff5 Revert "HBASE-22890 Verify the files when RegionServer is
starting and BucketCache is in file mode"
3b6cff5 is described below
commit 3b6cff590e99245cd972193d32ef73618cce41b3
Author: Reid Chan
AuthorDate: Mon Sep 16 17:50:57 2019 +0800
Revert "HBASE-22890 Verify the files when RegionServer is starting and
BucketCache is in file mode"
Reason: There're still some concerns on whether to delete cached data file.
This reverts commit 5bf60ec55fdf637d80492b61e6e6d8d605c5ef4a.
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 75 ++
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 152 +--
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 59
.../hbase/io/hfile/bucket/TestFileIOEngine.java| 2 +-
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 297 -
5 files changed, 32 insertions(+), 553 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 98abfc8..c5a1b21 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -29,7 +29,6 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -70,7 +69,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -244,17 +242,6 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
- private String ioEngineName;
- private static final String FILE_VERIFY_ALGORITHM =
-"hbase.bucketcache.persistent.file.integrity.check.algorithm";
- private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
-
- /**
- * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
- * persistent file integrity, default algorithm is MD5
- * */
- private String algorithm;
-
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -265,7 +252,8 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws IOException {
+ throws FileNotFoundException, IOException {
+this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -287,7 +275,6 @@ public class BucketCache implements BlockCache, HeapSize {
", memoryFactor: " + memoryFactor);
this.cacheCapacity = capacity;
-this.ioEngineName = ioEngineName;
this.persistencePath = persistencePath;
this.blockSize = blockSize;
this.ioErrorsTolerationDuration = ioErrorsTolerationDuration;
@@ -301,15 +288,14 @@ public class BucketCache implements BlockCache, HeapSize {
this.ramCache = new ConcurrentHashMap();
this.backingMap = new ConcurrentHashMap((int)
blockNumCapacity);
-this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
-ioEngine = getIOEngineFromName();
+
if (ioEngine.isPersistent() && persistencePath != null) {
try {
retrieveFromFile(bucketSizes);
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
th
[hbase] branch branch-1.4 updated: Revert "HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode"
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new a31cd0c Revert "HBASE-22890 Verify the files when RegionServer is
starting and BucketCache is in file mode"
a31cd0c is described below
commit a31cd0c0df9705db41551fe3403d5aef0da20033
Author: Reid Chan
AuthorDate: Mon Sep 16 17:50:57 2019 +0800
Revert "HBASE-22890 Verify the files when RegionServer is starting and
BucketCache is in file mode"
Reason: There're still some concerns on whether to delete cached data file.
This reverts commit 5bf60ec55fdf637d80492b61e6e6d8d605c5ef4a.
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 75 ++
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 152 +--
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 59
.../hbase/io/hfile/bucket/TestFileIOEngine.java| 2 +-
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 297 -
5 files changed, 32 insertions(+), 553 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index af10f2e..5a4ac13 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -29,7 +29,6 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -70,7 +69,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -244,17 +242,6 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
- private String ioEngineName;
- private static final String FILE_VERIFY_ALGORITHM =
-"hbase.bucketcache.persistent.file.integrity.check.algorithm";
- private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
-
- /**
- * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
- * persistent file integrity, default algorithm is MD5
- * */
- private String algorithm;
-
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -265,7 +252,8 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws IOException {
+ throws FileNotFoundException, IOException {
+this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -287,7 +275,6 @@ public class BucketCache implements BlockCache, HeapSize {
", memoryFactor: " + memoryFactor);
this.cacheCapacity = capacity;
-this.ioEngineName = ioEngineName;
this.persistencePath = persistencePath;
this.blockSize = blockSize;
this.ioErrorsTolerationDuration = ioErrorsTolerationDuration;
@@ -301,15 +288,14 @@ public class BucketCache implements BlockCache, HeapSize {
this.ramCache = new ConcurrentHashMap();
this.backingMap = new ConcurrentHashMap((int)
blockNumCapacity);
-this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
-ioEngine = getIOEngineFromName();
+
if (ioEngine.isPersistent() && persistencePath != null) {
try {
retrieveFromFile(bucketSizes);
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
th
[hbase] branch branch-1 updated: HBASE-22890 Verify the file integrity in persistent IOEngine
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new b5b871c HBASE-22890 Verify the file integrity in persistent IOEngine
b5b871c is described below
commit b5b871c13397acbdea82c091f5e2fd43e95e3b13
Author: zbq.dean
AuthorDate: Fri Sep 20 14:09:34 2019 +0800
HBASE-22890 Verify the file integrity in persistent IOEngine
Signed-off-by Anoop Sam John
Signed-off-by stack
Signed-off-by Reid Chan
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 81 --
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 88 ++-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 44
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 282 +
4 files changed, 473 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index c5a1b21..1e87a8e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -69,6 +69,8 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -242,6 +244,16 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -252,8 +264,9 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws FileNotFoundException, IOException {
-this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
+ throws IOException {
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
+ioEngine = getIOEngineFromName(ioEngineName, capacity);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -295,7 +308,7 @@ public class BucketCache implements BlockCache, HeapSize {
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
throw new RuntimeException(cnfe);
}
}
@@ -1021,41 +1034,69 @@ public class BucketCache implements BlockCache,
HeapSize {
private void persistToFile() throws IOException {
assert !cacheEnabled;
-FileOutputStream fos = null;
-ObjectOutputStream oos = null;
-try {
+try (ObjectOutputStream oos = new ObjectOutputStream(
+ new FileOutputStream(persistencePath, false))){
if (!ioEngine.isPersistent()) {
throw new IOException("Attempt to persist non-persistent cache
mappings!");
}
- fos = new FileOutputStream(persistencePath, false);
- oos = new ObjectOutputStream(fos);
+ byte[] checksum = ((PersistentIOEngine)
ioEngine).calculateChecksum(algorithm);
+ if (checksum != null) {
+oos.write(ProtobufUtil.PB_MAGIC);
+oos.writeInt(checksum.length);
+oos.write(checksum);
+ }
oos.writeLong(cacheCapacity);
oos.writeUTF(ioEngine.getClass().getName());
oos.writeUTF(backingMap.getClass().getName());
oos.writeObject(deserialiserMap);
oos.
[hbase] branch branch-1.4 updated: HBASE-22890 Verify the file integrity in persistent IOEngine
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 29080ed HBASE-22890 Verify the file integrity in persistent IOEngine
29080ed is described below
commit 29080eda9859a3689910eab285a3e51481f772ff
Author: zbq.dean
AuthorDate: Fri Sep 20 14:09:34 2019 +0800
HBASE-22890 Verify the file integrity in persistent IOEngine
Signed-off-by Anoop Sam John
Signed-off-by stack
Signed-off-by Reid Chan
---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 81 --
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 88 ++-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 44
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 282 +
4 files changed, 473 insertions(+), 22 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 5a4ac13..5c02166 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -69,6 +69,8 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -242,6 +244,16 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
FileNotFoundException,
IOException {
@@ -252,8 +264,9 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String
persistencePath, int ioErrorsTolerationDuration,
Configuration conf)
- throws FileNotFoundException, IOException {
-this.ioEngine = getIOEngineFromName(ioEngineName, capacity);
+ throws IOException {
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
+ioEngine = getIOEngineFromName(ioEngineName, capacity);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
if (blockNumCapacity >= Integer.MAX_VALUE) {
@@ -295,7 +308,7 @@ public class BucketCache implements BlockCache, HeapSize {
} catch (IOException ioex) {
LOG.error("Can't restore from file because of", ioex);
} catch (ClassNotFoundException cnfe) {
-LOG.error("Can't restore from file in rebuild because can't
deserialise",cnfe);
+LOG.error("Can't restore from file in rebuild because can't
deserialise", cnfe);
throw new RuntimeException(cnfe);
}
}
@@ -1021,41 +1034,69 @@ public class BucketCache implements BlockCache,
HeapSize {
private void persistToFile() throws IOException {
assert !cacheEnabled;
-FileOutputStream fos = null;
-ObjectOutputStream oos = null;
-try {
+try (ObjectOutputStream oos = new ObjectOutputStream(
+ new FileOutputStream(persistencePath, false))){
if (!ioEngine.isPersistent()) {
throw new IOException("Attempt to persist non-persistent cache
mappings!");
}
- fos = new FileOutputStream(persistencePath, false);
- oos = new ObjectOutputStream(fos);
+ byte[] checksum = ((PersistentIOEngine)
ioEngine).calculateChecksum(algorithm);
+ if (checksum != null) {
+oos.write(ProtobufUtil.PB_MAGIC);
+oos.writeInt(checksum.length);
+oos.write(checksum);
+ }
oos.writeLong(cacheCapacity);
oos.writeUTF(ioEngine.getClass().getName());
oos.writeUTF(backingMap.getClass().getName());
oos.writeObject(deserialiserMap);
[hbase] branch master updated: HBASE-22975 Add read and write QPS metrics at server level and table level (#615)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a8e3d23 HBASE-22975 Add read and write QPS metrics at server level
and table level (#615)
a8e3d23 is described below
commit a8e3d23cca4d98d1c288bca0da056f92c49b453f
Author: zbq.dean
AuthorDate: Mon Sep 23 12:51:25 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
(#615)
Signed-off-by Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 +++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 102 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 16
.../hbase/regionserver/MetricsRegionServer.java| 33 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 237 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..d0085ff
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 980388f..5a3f3b9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -20,6 +20,8 @@ import java.util.HashMap;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import org.apache.yetus.audience.InterfaceAudience;
@@ -171,4 +173,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.ja
[hbase] branch branch-1 updated: HBASE-22975 Add read and write QPS metrics at server level and table level
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 771e184 HBASE-22975 Add read and write QPS metrics at server level
and table level
771e184 is described below
commit 771e18437605c80ba37d8b126c2ffa668ca56805
Author: zbq.dean
AuthorDate: Mon Sep 23 13:51:51 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
Signed-off-by Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 ++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 107 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 20
.../hbase/regionserver/MetricsRegionServer.java| 34 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 247 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..fcce6e3
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 2c052f2..ec6e932 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import com.google.common.annotations.VisibleForTesting;
@@ -172,4 +174,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoo
[hbase] branch branch-1.4 updated: HBASE-22975 Add read and write QPS metrics at server level and table level
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 8fa1505 HBASE-22975 Add read and write QPS metrics at server level
and table level
8fa1505 is described below
commit 8fa1505ca4e0912c211551b6d7ae5cae9ab17a0d
Author: zbq.dean
AuthorDate: Mon Sep 23 13:51:51 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
Signed-off-by Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 ++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 107 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 20
.../hbase/regionserver/MetricsRegionServer.java| 34 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 247 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..fcce6e3
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 2c052f2..ec6e932 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import com.google.common.annotations.VisibleForTesting;
@@ -172,4 +174,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoo
[hbase] branch branch-2 updated: HBASE-22975 Add read and write QPS metrics at server level and table level
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 0c5b6df HBASE-22975 Add read and write QPS metrics at server level
and table level
0c5b6df is described below
commit 0c5b6df52e8c973d7993294d50028399ad66c1af
Author: zbq.dean
AuthorDate: Mon Sep 23 14:18:29 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
Signed-off-by: Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 +++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 102 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 16
.../hbase/regionserver/MetricsRegionServer.java| 34 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 238 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..d0085ff
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 980388f..5a3f3b9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -20,6 +20,8 @@ import java.util.HashMap;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import org.apache.yetus.audience.InterfaceAudience;
@@ -171,4 +173,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java
b/hbas
[hbase] branch branch-2.1 updated: HBASE-22975 Add read and write QPS metrics at server level and table level
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e040010 HBASE-22975 Add read and write QPS metrics at server level
and table level
e040010 is described below
commit e040010ed0391d29b36bc088dbaba76502bd85f2
Author: zbq.dean
AuthorDate: Mon Sep 23 14:18:29 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
Signed-off-by: Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 +++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 102 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 16
.../hbase/regionserver/MetricsRegionServer.java| 34 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 238 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..d0085ff
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 980388f..5a3f3b9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -20,6 +20,8 @@ import java.util.HashMap;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import org.apache.yetus.audience.InterfaceAudience;
@@ -171,4 +173,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java
b/
[hbase] branch branch-2.2 updated: HBASE-22975 Add read and write QPS metrics at server level and table level
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new fd4cf24 HBASE-22975 Add read and write QPS metrics at server level
and table level
fd4cf24 is described below
commit fd4cf240c3896e1e29bca2c18859845a722b4ad7
Author: zbq.dean
AuthorDate: Mon Sep 23 14:18:29 2019 +0800
HBASE-22975 Add read and write QPS metrics at server level and table level
Signed-off-by: Reid Chan
---
.../hbase/regionserver/MetricsTableQueryMeter.java | 53 +++
.../regionserver/MetricsTableLatenciesImpl.java| 13 +++
.../regionserver/MetricsTableQueryMeterImpl.java | 102 +
.../apache/hadoop/hbase/regionserver/HRegion.java | 16
.../hbase/regionserver/MetricsRegionServer.java| 34 +++
.../regionserver/RegionServerTableMetrics.java | 20
6 files changed, 238 insertions(+)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
new file mode 100644
index 000..d0085ff
--- /dev/null
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Query Per Second for each table in a RegionServer.
+ */
+@InterfaceAudience.Private
+public interface MetricsTableQueryMeter {
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableReadQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table read QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableReadQueryMeter(TableName tableName);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ * @param count Number of occurrences to record
+ */
+ void updateTableWriteQueryMeter(TableName tableName, long count);
+
+ /**
+ * Update table write QPS
+ * @param tableName The table the metric is for
+ */
+ void updateTableWriteQueryMeter(TableName tableName);
+}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 980388f..5a3f3b9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -20,6 +20,8 @@ import java.util.HashMap;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import org.apache.yetus.audience.InterfaceAudience;
@@ -171,4 +173,15 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(String tableName, long t) {
getOrCreateTableHistogram(tableName).updateScanTime(t);
}
+
+ @Override
+ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
+MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
+// source is registered in supers constructor, sometimes called before the
whole initialization.
+metricsRegistry.snapshot(mrb, all);
+if (metricsAdapter != null) {
+ // snapshot MetricRegistry as well
+ metricsAdapter.snapshotAllMetrics(registry, mrb);
+}
+ }
}
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java
b/
[hbase] branch 1.4 deleted (was a8e3d23)
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a change to branch 1.4 in repository https://gitbox.apache.org/repos/asf/hbase.git. was a8e3d23 HBASE-22975 Add read and write QPS metrics at server level and table level (#615) The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[hbase] branch master updated: HBASE-23017 Verify the file integrity in persistent IOEngine
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 16da123 HBASE-23017 Verify the file integrity in persistent IOEngine
16da123 is described below
commit 16da123df45af712f604cff32897d6c1166b86b4
Author: Baiqiang Zhao
AuthorDate: Fri Oct 11 14:38:00 2019 +0800
HBASE-23017 Verify the file integrity in persistent IOEngine
Signed-off-by: Anoop Sam John
Signed-off-by: Reid Chan
---
.../src/main/protobuf/BucketCacheEntry.proto | 1 +
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 22 ++
.../hbase/io/hfile/bucket/BucketProtoUtils.java| 14 +-
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 12 +-
.../hbase/io/hfile/bucket/FileMmapIOEngine.java| 11 +-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 116 ++
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 247 +
7 files changed, 411 insertions(+), 12 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
index d78acc0..038c6ca 100644
--- a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
@@ -31,6 +31,7 @@ message BucketCacheEntry {
required string map_class = 3;
map deserializers = 4;
required BackingMap backing_map = 5;
+ optional bytes checksum = 6;
}
message BackingMap {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 99abfea..7d8f582 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -238,6 +238,16 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
IOException {
this(ioEngineName, capacity, blockSize, bucketSizes, writerThreadNum,
writerQLen,
@@ -247,6 +257,7 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath, int
ioErrorsTolerationDuration,
Configuration conf) throws IOException {
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
this.ioEngine = getIOEngineFromName(ioEngineName, capacity,
persistencePath);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
@@ -1131,6 +1142,13 @@ public class BucketCache implements BlockCache, HeapSize
{
}
private void parsePB(BucketCacheProtos.BucketCacheEntry proto) throws
IOException {
+if (proto.hasChecksum()) {
+ ((PersistentIOEngine)
ioEngine).verifyFileIntegrity(proto.getChecksum().toByteArray(),
+algorithm);
+} else {
+ // if has not checksum, it means the persistence file is old format
+ LOG.info("Persistent file is old format, it does not support verifying
file integrity!");
+}
verifyCapacityAndClasses(proto.getCacheCapacity(), proto.getIoClass(),
proto.getMapClass());
backingMap = BucketProtoUtils.fromPB(proto.getDeserializersMap(),
proto.getBackingMap());
}
@@ -1235,6 +1253,10 @@ public class BucketCache implements BlockCache, HeapSize
{
return this.bucketAllocator.getUsedSize();
}
+ protected String getAlgorithm() {
+return algorithm;
+ }
+
/**
* Evicts all blocks for a specific HFile.
*
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
index 69b8370..f3d63d4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockPriority;
import org.apache.hadoo
[hbase] branch branch-2 updated: HBASE-23017 Verify the file integrity in persistent IOEngine
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 273c0ba HBASE-23017 Verify the file integrity in persistent IOEngine
273c0ba is described below
commit 273c0ba69fa681b543513a1a4cb1fed052d11bba
Author: Baiqiang Zhao
AuthorDate: Fri Oct 11 14:38:00 2019 +0800
HBASE-23017 Verify the file integrity in persistent IOEngine
Signed-off-by: Anoop Sam John
Signed-off-by: Reid Chan
---
.../src/main/protobuf/BucketCacheEntry.proto | 1 +
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 22 ++
.../hbase/io/hfile/bucket/BucketProtoUtils.java| 14 +-
.../hadoop/hbase/io/hfile/bucket/FileIOEngine.java | 12 +-
.../hbase/io/hfile/bucket/FileMmapIOEngine.java| 11 +-
.../hbase/io/hfile/bucket/PersistentIOEngine.java | 116 ++
.../io/hfile/bucket/TestVerifyBucketCacheFile.java | 247 +
7 files changed, 411 insertions(+), 12 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
index d78acc0..038c6ca 100644
--- a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto
@@ -31,6 +31,7 @@ message BucketCacheEntry {
required string map_class = 3;
map deserializers = 4;
required BackingMap backing_map = 5;
+ optional bytes checksum = 6;
}
message BackingMap {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 99abfea..7d8f582 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -238,6 +238,16 @@ public class BucketCache implements BlockCache, HeapSize {
/** In-memory bucket size */
private float memoryFactor;
+ private static final String FILE_VERIFY_ALGORITHM =
+"hbase.bucketcache.persistent.file.integrity.check.algorithm";
+ private static final String DEFAULT_FILE_VERIFY_ALGORITHM = "MD5";
+
+ /**
+ * Use {@link java.security.MessageDigest} class's encryption algorithms to
check
+ * persistent file integrity, default algorithm is MD5
+ * */
+ private String algorithm;
+
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath) throws
IOException {
this(ioEngineName, capacity, blockSize, bucketSizes, writerThreadNum,
writerQLen,
@@ -247,6 +257,7 @@ public class BucketCache implements BlockCache, HeapSize {
public BucketCache(String ioEngineName, long capacity, int blockSize, int[]
bucketSizes,
int writerThreadNum, int writerQLen, String persistencePath, int
ioErrorsTolerationDuration,
Configuration conf) throws IOException {
+this.algorithm = conf.get(FILE_VERIFY_ALGORITHM,
DEFAULT_FILE_VERIFY_ALGORITHM);
this.ioEngine = getIOEngineFromName(ioEngineName, capacity,
persistencePath);
this.writerThreads = new WriterThread[writerThreadNum];
long blockNumCapacity = capacity / blockSize;
@@ -1131,6 +1142,13 @@ public class BucketCache implements BlockCache, HeapSize
{
}
private void parsePB(BucketCacheProtos.BucketCacheEntry proto) throws
IOException {
+if (proto.hasChecksum()) {
+ ((PersistentIOEngine)
ioEngine).verifyFileIntegrity(proto.getChecksum().toByteArray(),
+algorithm);
+} else {
+ // if has not checksum, it means the persistence file is old format
+ LOG.info("Persistent file is old format, it does not support verifying
file integrity!");
+}
verifyCapacityAndClasses(proto.getCacheCapacity(), proto.getIoClass(),
proto.getMapClass());
backingMap = BucketProtoUtils.fromPB(proto.getDeserializersMap(),
proto.getBackingMap());
}
@@ -1235,6 +1253,10 @@ public class BucketCache implements BlockCache, HeapSize
{
return this.bucketAllocator.getUsedSize();
}
+ protected String getAlgorithm() {
+return algorithm;
+ }
+
/**
* Evicts all blocks for a specific HFile.
*
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
index 69b8370..f3d63d4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockPriority;
import org.apache.hadoo
[hbase] branch master updated: HBASE-23144 Compact_rs throw wrong number of arguments
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new f0b2212 HBASE-23144 Compact_rs throw wrong number of arguments f0b2212 is described below commit f0b22120a09f27c730b4a06c34a9b3f24433bd49 Author: Karthik Palanisamy AuthorDate: Fri Oct 11 00:08:57 2019 -0700 HBASE-23144 Compact_rs throw wrong number of arguments Signed-off-by: Reid Chan --- hbase-shell/src/main/ruby/hbase/admin.rb | 6 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 10 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index b854eaf..5f4f16d 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -115,7 +115,11 @@ module Hbase # Requests to compact all regions on the regionserver def compact_regionserver(servername, major = false) - @admin.compactRegionServer(ServerName.valueOf(servername), major) + if major +@admin.majorCompactRegionServer(ServerName.valueOf(servername)) + else +@admin.compactRegionServer(ServerName.valueOf(servername)) + end end #-- diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index 1461c7f..e001445 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -107,6 +107,16 @@ module Hbase command(:flush, s.toString) end end + #--- +define_test 'compact all regions by server name' do + servers = admin.list_liveservers + servers.each do |s| +command(:compact_rs, s.to_s) +# major compact +command(:compact_rs, s.to_s, true) +break + end +end #---
[hbase] branch master updated: HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and retrieve from file
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 5848e14 HBASE-23056 Block count is 0 when BucketCache using
persistent IOEngine and retrieve from file
5848e14 is described below
commit 5848e149f43c2b176baa25c0fa14744a88c9c217
Author: zbq.dean
AuthorDate: Fri Sep 20 17:28:43 2019 +0800
HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and
retrieve from file
Signed-off-by: Reid Chan
---
.../main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 7d8f582..790b93d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1094,6 +1094,7 @@ public class BucketCache implements BlockCache, HeapSize {
}
parsePB(BucketCacheProtos.BucketCacheEntry.parseDelimitedFrom(in));
bucketAllocator = new BucketAllocator(cacheCapacity, bucketSizes,
backingMap, realCacheSize);
+ blockNumber.add(backingMap.size());
}
}
[hbase] branch branch-2 updated: HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and retrieve from file
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 595bcda HBASE-23056 Block count is 0 when BucketCache using
persistent IOEngine and retrieve from file
595bcda is described below
commit 595bcda9c34fa6e24b6cce2ab4be4ee057983b1f
Author: zbq.dean
AuthorDate: Fri Sep 20 17:28:43 2019 +0800
HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and
retrieve from file
Signed-off-by: Reid Chan
---
.../main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 7d8f582..790b93d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1094,6 +1094,7 @@ public class BucketCache implements BlockCache, HeapSize {
}
parsePB(BucketCacheProtos.BucketCacheEntry.parseDelimitedFrom(in));
bucketAllocator = new BucketAllocator(cacheCapacity, bucketSizes,
backingMap, realCacheSize);
+ blockNumber.add(backingMap.size());
}
}
[hbase] branch branch-1 updated: HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and retrieve from file
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 4a0442d HBASE-23056 Block count is 0 when BucketCache using
persistent IOEngine and retrieve from file
4a0442d is described below
commit 4a0442d14eddc509fb5ca6c5a369c3a7c1b25064
Author: zbq.dean
AuthorDate: Fri Sep 20 17:31:13 2019 +0800
HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and
retrieve from file
Signed-off-by: Reid Chan
---
.../main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index daf96ef..041179a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1120,6 +1120,7 @@ public class BucketCache implements BlockCache, HeapSize {
bucketAllocator = allocator;
deserialiserMap = deserMap;
backingMap = backingMapFromFile;
+ blockNumber.set(backingMap.size());
} finally {
if (ois != null) {
ois.close();
[hbase] branch branch-1.4 updated: HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and retrieve from file
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new a7fff50 HBASE-23056 Block count is 0 when BucketCache using
persistent IOEngine and retrieve from file
a7fff50 is described below
commit a7fff5043a233ce649b8707059be906f9344b9a3
Author: zbq.dean
AuthorDate: Fri Sep 20 17:31:13 2019 +0800
HBASE-23056 Block count is 0 when BucketCache using persistent IOEngine and
retrieve from file
Signed-off-by: Reid Chan
---
.../main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 5c241f4..40d6602 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1120,6 +1120,7 @@ public class BucketCache implements BlockCache, HeapSize {
bucketAllocator = allocator;
deserialiserMap = deserMap;
backingMap = backingMapFromFile;
+ blockNumber.set(backingMap.size());
} finally {
if (ois != null) {
ois.close();
[hbase] branch master updated: HBASE-21048 Get LogLevel is not working from console in secure environment
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 16146a1 HBASE-21048 Get LogLevel is not working from console in
secure environment
16146a1 is described below
commit 16146a18394e25c6c83c97812e7cb16db4166bf7
Author: Wei-Chiu Chuang
AuthorDate: Sat Apr 13 18:58:35 2019 +0800
HBASE-21048 Get LogLevel is not working from console in secure environment
Signed-off-by: Reid Chan
Amend author: Reid Chan
---
hbase-http/pom.xml | 5 +
.../org/apache/hadoop/hbase/http/log/LogLevel.java | 211 ++--
.../apache/hadoop/hbase/http/log/TestLogLevel.java | 352 ++---
3 files changed, 495 insertions(+), 73 deletions(-)
diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml
index 985d75f..65d4f5b 100644
--- a/hbase-http/pom.xml
+++ b/hbase-http/pom.xml
@@ -298,6 +298,11 @@
mockito-core
test
+
+ org.apache.hadoop
+ hadoop-minikdc
+ test
+
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 6f619ae..7182a0b 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -26,61 +26,230 @@ import java.net.URL;
import java.net.URLConnection;
import java.util.Objects;
import java.util.regex.Pattern;
+
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.logging.impl.Jdk14Logger;
import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
import org.apache.log4j.LogManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.impl.Log4jLoggerAdapter;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
+
/**
* Change log level in runtime.
*/
@InterfaceAudience.Private
public final class LogLevel {
private static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel ]\n"
- + "\t[-setlevel ]\n";
+ + "\t[-getlevel \n"
+ + "\t[-setlevel";
+ public static final String PROTOCOL_HTTP = "http";
/**
* A command line implementation
*/
- public static void main(String[] args) {
-if (args.length == 3 && "-getlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]);
- return;
-}
-else if (args.length == 4 && "-setlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]
- + "&level=" + args[3]);
- return;
-}
+ public static void main(String[] args) throws Exception {
+CLI cli = new CLI(new Configuration());
+System.exit(cli.run(args));
+ }
+ /**
+ * Valid command line options.
+ */
+ private enum Operations {
+GETLEVEL,
+SETLEVEL,
+UNKNOWN
+ }
+
+ private static void printUsage() {
System.err.println(USAGES);
System.exit(-1);
}
- private static void process(String urlstring) {
-try {
- URL url = new URL(urlstring);
- System.out.println("Connecting to " + url);
- URLConnection connection = url.openConnection();
+ @VisibleForTesting
+ static class CLI extends Configured implements Tool {
+private Operations operation = Operations.UNKNOWN;
+private String hostName;
+private String className;
+private String level;
+
+CLI(Configuration conf) {
+ setConf(conf);
+}
+
+@Override
+public int run(String[] args) throws Exception {
+ try {
+parseArguments(args);
+sendLogLevelRequest();
+ } catch (HadoopIllegalArgumentException e) {
+printUsage();
+ }
+ return 0;
+}
+
+/**
+ * Send HTTP request to the daemon.
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exce
[hbase] branch branch-2 updated: HBASE-22240 [backport] HBASE-19762 Fix Checkstyle errors in hbase-http
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 0da8b2c HBASE-22240 [backport] HBASE-19762 Fix Checkstyle errors in
hbase-http
0da8b2c is described below
commit 0da8b2ce135ca20cdfe287cadd955362bb4457b4
Author: Jan Hentschel
AuthorDate: Wed Feb 7 18:52:59 2018 +0100
HBASE-22240 [backport] HBASE-19762 Fix Checkstyle errors in hbase-http
Signed-off-by: Reid Chan
---
.../resources/hbase/checkstyle-suppressions.xml| 7 +-
hbase-http/pom.xml | 7 ++
.../hbase/http/ClickjackingPreventionFilter.java | 45
.../org/apache/hadoop/hbase/http/HtmlQuoting.java | 56 ++
.../org/apache/hadoop/hbase/http/HttpConfig.java | 5 +-
.../hadoop/hbase/http/HttpRequestLogAppender.java | 2 +-
.../org/apache/hadoop/hbase/http/HttpServer.java | 120 ++---
.../org/apache/hadoop/hbase/http/InfoServer.java | 51 -
.../apache/hadoop/hbase/http/NoCacheFilter.java| 8 +-
.../apache/hadoop/hbase/http/ProfileServlet.java | 4 +-
.../apache/hadoop/hbase/http/conf/ConfServlet.java | 7 +-
.../org/apache/hadoop/hbase/http/log/LogLevel.java | 30 +++---
.../org/apache/hadoop/hbase/util/ProcessUtils.java | 4 +-
.../hbase/http/HttpServerFunctionalTest.java | 27 ++---
.../apache/hadoop/hbase/http/TestGlobalFilter.java | 44
.../apache/hadoop/hbase/http/TestHtmlQuoting.java | 7 +-
.../apache/hadoop/hbase/http/TestHttpServer.java | 103 --
.../apache/hadoop/hbase/http/TestPathFilter.java | 41 ---
.../hadoop/hbase/http/TestServletFilter.java | 38 +++
.../hadoop/hbase/http/TestSpnegoHttpServer.java| 59 +-
.../apache/hadoop/hbase/http/log/TestLogLevel.java | 18 ++--
.../hadoop/hbase/http/resource/JerseyResource.java | 2 +-
.../hadoop/hbase/http/ssl/KeyStoreTestUtil.java| 13 ++-
23 files changed, 376 insertions(+), 322 deletions(-)
diff --git
a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
index 77eefc2..090f1d9 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
@@ -1,7 +1,7 @@
https://checkstyle.org/dtds/suppressions_1_2.dtd";>
+"-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
+"https://checkstyle.org/dtds/suppressions_1_2.dtd";>
";
- static final Pattern TAG = Pattern.compile("<[^>]*>");
+ private static final String MARKER = "";
+ private static final Pattern TAG = Pattern.compile("<[^>]*>");
/**
* A servlet implementation
@@ -98,9 +96,8 @@ public final class LogLevel {
private static final long serialVersionUID = 1L;
@Override
-public void doGet(HttpServletRequest request, HttpServletResponse response
-) throws ServletException, IOException {
-
+public void doGet(HttpServletRequest request, HttpServletResponse response)
+throws ServletException, IOException {
// Do the authorization
if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
response)) {
@@ -175,8 +172,7 @@ public final class LogLevel {
+ "Set the specified log level for the specified log name." +
"\n" + "\n"
+ "\n" + "\n" + "\n" + "\n" + "\n";
-private static void process(org.apache.log4j.Logger log, String level,
-PrintWriter out) throws IOException {
+private static void process(org.apache.log4j.Logger log, String level,
PrintWriter out) {
if (level != null) {
if (!level.equals(org.apache.log4j.Level.toLevel(level).toString())) {
out.println(MARKER + "" + "Bad level :
" + level
@@ -192,14 +188,18 @@ public final class LogLevel {
}
private static void process(java.util.logging.Logger log, String level,
-PrintWriter out) throws IOException {
+PrintWriter out) {
if (level != null) {
log.setLevel(java.util.logging.Level.parse(level));
out.println(MARKER + "Setting Level to " + level + " ...");
}
java.util.logging.Level lev;
- for(; (lev = log.getLevel()) == null; log = log.getParent());
+
+ while ((lev = log.getLevel()) == null) {
+log = log.getParent();
+ }
+
out.println(MARKER + "Effective level: " + lev + "");
}
}
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/util/ProcessUtils.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/u
[hbase] branch branch-2 updated: HBASE-21048 Get LogLevel is not working from console in secure environment
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new dca30ce HBASE-21048 Get LogLevel is not working from console in
secure environment
dca30ce is described below
commit dca30ce620c7704913a6f0ea9a3c5c2194a09468
Author: Wei-Chiu Chuang
AuthorDate: Sat Apr 13 18:58:35 2019 +0800
HBASE-21048 Get LogLevel is not working from console in secure environment
Signed-off-by: Reid Chan
Amend author: Reid Chan
---
hbase-http/pom.xml | 5 +
.../org/apache/hadoop/hbase/http/log/LogLevel.java | 211 ++--
.../apache/hadoop/hbase/http/log/TestLogLevel.java | 352 ++---
3 files changed, 495 insertions(+), 73 deletions(-)
diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml
index ada1892..d607161 100644
--- a/hbase-http/pom.xml
+++ b/hbase-http/pom.xml
@@ -298,6 +298,11 @@
mockito-core
test
+
+ org.apache.hadoop
+ hadoop-minikdc
+ test
+
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 6f619ae..7182a0b 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -26,61 +26,230 @@ import java.net.URL;
import java.net.URLConnection;
import java.util.Objects;
import java.util.regex.Pattern;
+
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.logging.impl.Jdk14Logger;
import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
import org.apache.log4j.LogManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.impl.Log4jLoggerAdapter;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
+
/**
* Change log level in runtime.
*/
@InterfaceAudience.Private
public final class LogLevel {
private static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel ]\n"
- + "\t[-setlevel ]\n";
+ + "\t[-getlevel \n"
+ + "\t[-setlevel";
+ public static final String PROTOCOL_HTTP = "http";
/**
* A command line implementation
*/
- public static void main(String[] args) {
-if (args.length == 3 && "-getlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]);
- return;
-}
-else if (args.length == 4 && "-setlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]
- + "&level=" + args[3]);
- return;
-}
+ public static void main(String[] args) throws Exception {
+CLI cli = new CLI(new Configuration());
+System.exit(cli.run(args));
+ }
+ /**
+ * Valid command line options.
+ */
+ private enum Operations {
+GETLEVEL,
+SETLEVEL,
+UNKNOWN
+ }
+
+ private static void printUsage() {
System.err.println(USAGES);
System.exit(-1);
}
- private static void process(String urlstring) {
-try {
- URL url = new URL(urlstring);
- System.out.println("Connecting to " + url);
- URLConnection connection = url.openConnection();
+ @VisibleForTesting
+ static class CLI extends Configured implements Tool {
+private Operations operation = Operations.UNKNOWN;
+private String hostName;
+private String className;
+private String level;
+
+CLI(Configuration conf) {
+ setConf(conf);
+}
+
+@Override
+public int run(String[] args) throws Exception {
+ try {
+parseArguments(args);
+sendLogLevelRequest();
+ } catch (HadoopIllegalArgumentException e) {
+printUsage();
+ }
+ return 0;
+}
+
+/**
+ * Send HTTP request to the daemon.
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exce
[hbase] branch branch-1 updated: HBASE-21048 Get LogLevel is not working from console in secure environment
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 4af4afc HBASE-21048 Get LogLevel is not working from console in
secure environment
4af4afc is described below
commit 4af4afc94f2836400b716dfaeef2c661064eb4fe
Author: Wei-Chiu Chuang
AuthorDate: Tue Apr 16 13:58:46 2019 -0700
HBASE-21048 Get LogLevel is not working from console in secure environment
Signed-off-by: Reid Chan
---
.../org/apache/hadoop/hbase/http/log/LogLevel.java | 236 +++--
.../apache/hadoop/hbase/http/log/TestLogLevel.java | 374 +
2 files changed, 514 insertions(+), 96 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 7701a25..328e1b1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.http.log;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -34,59 +37,223 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Jdk14Logger;
import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
/**
* Change log level in runtime.
*/
@InterfaceStability.Evolving
public class LogLevel {
- public static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel ]\n"
- + "\t[-setlevel ]\n";
+ private static final String USAGES = "\nUsage: General options are:\n"
+ + "\t[-getlevel \n"
+ + "\t[-setlevel";
+ public static final String PROTOCOL_HTTP = "http";
/**
* A command line implementation
*/
- public static void main(String[] args) {
-if (args.length == 3 && "-getlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]);
- return;
-}
-else if (args.length == 4 && "-setlevel".equals(args[0])) {
- process("http://"; + args[1] + "/logLevel?log=" + args[2]
- + "&level=" + args[3]);
- return;
-}
+ public static void main(String[] args) throws Exception {
+CLI cli = new CLI(new Configuration());
+System.exit(cli.run(args));
+ }
+
+ /**
+ * Valid command line options.
+ */
+ private enum Operations {
+GETLEVEL,
+SETLEVEL,
+UNKNOWN
+ }
+ private static void printUsage() {
System.err.println(USAGES);
System.exit(-1);
}
- private static void process(String urlstring) {
-try {
- URL url = new URL(urlstring);
- System.out.println("Connecting to " + url);
- URLConnection connection = url.openConnection();
+ @VisibleForTesting
+ static class CLI extends Configured implements Tool {
+private Operations operation = Operations.UNKNOWN;
+private String hostName;
+private String className;
+private String level;
+
+CLI(Configuration conf) {
+ setConf(conf);
+}
+
+@Override
+public int run(String[] args) throws Exception {
+ try {
+parseArguments(args);
+sendLogLevelRequest();
+ } catch (HadoopIllegalArgumentException e) {
+printUsage();
+ }
+ return 0;
+}
+
+/**
+ * Send HTTP request to the daemon.
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+private void sendLogLevelRequest()
+throws HadoopIllegalArgumentException, Exception {
+ switch (operation) {
+case GETLEVEL:
+ doGetLevel();
+ break;
+case SETLEVEL:
+ doSetLevel();
+ break;
+default:
+ throw new HadoopIllegalArgumentException(
+ "Expect either -getlevel or -setlevel");
+ }
+}
+
+public void parseArguments(String[] ar
[hbase] branch branch-1 updated: HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 2ff4e4f HBASE-22559 [RPC] set guard against
CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
2ff4e4f is described below
commit 2ff4e4f09630a8c04e57d16a71362708dd165532
Author: Reid Chan
AuthorDate: Fri Jun 14 11:24:55 2019 +0800
HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
---
.../main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java | 14 ++
1 file changed, 14 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index 15c416c..d46786b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -133,6 +133,20 @@ public abstract class RpcExecutor {
this.abortable = abortable;
float callQueuesHandlersFactor =
this.conf.getFloat(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 0);
+if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0 ||
+Float.compare(0.0f, callQueuesHandlersFactor) > 0) {
+ LOG.warn(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY +
+" is *ILLEGAL*, it should be in range [0.0, 1.0]");
+ // For callQueuesHandlersFactor > 1.0, we just set it 1.0f.
+ if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0) {
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " 1.0f");
+callQueuesHandlersFactor = 1.0f;
+ } else {
+// But for callQueuesHandlersFactor < 0.0, following method
#computeNumCallQueues
+// will compute max(1, -x) => 1 which has same effect of default value.
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " default value
0.0f");
+ }
+}
this.numCallQueues = computeNumCallQueues(handlerCount,
callQueuesHandlersFactor);
this.queues = new ArrayList<>(this.numCallQueues);
[hbase] branch branch-1.4 updated: HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 3a8f7c9 HBASE-22559 [RPC] set guard against
CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
3a8f7c9 is described below
commit 3a8f7c9a3642ffca859bd6f363ff3831a459e7e2
Author: Reid Chan
AuthorDate: Fri Jun 14 11:24:55 2019 +0800
HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
Signed-off-by Andrew Purtell
---
.../main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java | 14 ++
1 file changed, 14 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index 15c416c..d46786b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -133,6 +133,20 @@ public abstract class RpcExecutor {
this.abortable = abortable;
float callQueuesHandlersFactor =
this.conf.getFloat(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 0);
+if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0 ||
+Float.compare(0.0f, callQueuesHandlersFactor) > 0) {
+ LOG.warn(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY +
+" is *ILLEGAL*, it should be in range [0.0, 1.0]");
+ // For callQueuesHandlersFactor > 1.0, we just set it 1.0f.
+ if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0) {
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " 1.0f");
+callQueuesHandlersFactor = 1.0f;
+ } else {
+// But for callQueuesHandlersFactor < 0.0, following method
#computeNumCallQueues
+// will compute max(1, -x) => 1 which has same effect of default value.
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " default value
0.0f");
+ }
+}
this.numCallQueues = computeNumCallQueues(handlerCount,
callQueuesHandlersFactor);
this.queues = new ArrayList<>(this.numCallQueues);
[hbase] branch master updated: HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new ab44531 HBASE-22559 [RPC] set guard against
CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
ab44531 is described below
commit ab4453158a41e29bb75e219079584997be756cbe
Author: Reid Chan
AuthorDate: Fri Jun 14 11:35:34 2019 +0800
HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
Signed-off-by: Andrew Purtell
---
.../main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java | 14 ++
1 file changed, 14 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index f63b243..3de5fa1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -116,6 +116,20 @@ public abstract class RpcExecutor {
this.abortable = abortable;
float callQueuesHandlersFactor =
this.conf.getFloat(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 0.1f);
+if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0 ||
+Float.compare(0.0f, callQueuesHandlersFactor) > 0) {
+ LOG.warn(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY +
+" is *ILLEGAL*, it should be in range [0.0, 1.0]");
+ // For callQueuesHandlersFactor > 1.0, we just set it 1.0f.
+ if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0) {
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " 1.0f");
+callQueuesHandlersFactor = 1.0f;
+ } else {
+// But for callQueuesHandlersFactor < 0.0, following method
#computeNumCallQueues
+// will compute max(1, -x) => 1 which has same effect of default value.
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " default value
0.0f");
+ }
+}
this.numCallQueues = computeNumCallQueues(handlerCount,
callQueuesHandlersFactor);
this.queues = new ArrayList<>(this.numCallQueues);
[hbase] branch branch-2 updated: HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new e2d8911 HBASE-22559 [RPC] set guard against
CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
e2d8911 is described below
commit e2d891172a0fa03e44d7e7ca5e8cce9435940d1e
Author: Reid Chan
AuthorDate: Fri Jun 14 11:35:34 2019 +0800
HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
Signed-off-by: Andrew Purtell
---
.../main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java | 14 ++
1 file changed, 14 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index 7470758..a532b6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -116,6 +116,20 @@ public abstract class RpcExecutor {
this.abortable = abortable;
float callQueuesHandlersFactor =
this.conf.getFloat(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 0.1f);
+if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0 ||
+Float.compare(0.0f, callQueuesHandlersFactor) > 0) {
+ LOG.warn(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY +
+" is *ILLEGAL*, it should be in range [0.0, 1.0]");
+ // For callQueuesHandlersFactor > 1.0, we just set it 1.0f.
+ if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0) {
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " 1.0f");
+callQueuesHandlersFactor = 1.0f;
+ } else {
+// But for callQueuesHandlersFactor < 0.0, following method
#computeNumCallQueues
+// will compute max(1, -x) => 1 which has same effect of default value.
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " default value
0.0f");
+ }
+}
this.numCallQueues = computeNumCallQueues(handlerCount,
callQueuesHandlersFactor);
this.queues = new ArrayList<>(this.numCallQueues);
[hbase] branch branch-2.2 updated: HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 0b17c7f HBASE-22559 [RPC] set guard against
CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
0b17c7f is described below
commit 0b17c7f7241d0d3e738d2ab234cf842823fe3480
Author: Reid Chan
AuthorDate: Fri Jun 14 11:35:34 2019 +0800
HBASE-22559 [RPC] set guard against CALL_QUEUE_HANDLER_FACTOR_CONF_KEY
Signed-off-by: Andrew Purtell
---
.../main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java | 14 ++
1 file changed, 14 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index 7470758..a532b6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -116,6 +116,20 @@ public abstract class RpcExecutor {
this.abortable = abortable;
float callQueuesHandlersFactor =
this.conf.getFloat(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 0.1f);
+if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0 ||
+Float.compare(0.0f, callQueuesHandlersFactor) > 0) {
+ LOG.warn(CALL_QUEUE_HANDLER_FACTOR_CONF_KEY +
+" is *ILLEGAL*, it should be in range [0.0, 1.0]");
+ // For callQueuesHandlersFactor > 1.0, we just set it 1.0f.
+ if (Float.compare(callQueuesHandlersFactor, 1.0f) > 0) {
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " 1.0f");
+callQueuesHandlersFactor = 1.0f;
+ } else {
+// But for callQueuesHandlersFactor < 0.0, following method
#computeNumCallQueues
+// will compute max(1, -x) => 1 which has same effect of default value.
+LOG.warn("Set " + CALL_QUEUE_HANDLER_FACTOR_CONF_KEY + " default value
0.0f");
+ }
+}
this.numCallQueues = computeNumCallQueues(handlerCount,
callQueuesHandlersFactor);
this.queues = new ArrayList<>(this.numCallQueues);
[hbase] branch branch-1 updated: HBASE-22562 Remove dead code: skipControl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 84af2a4 HBASE-22562 Remove dead code: skipControl
84af2a4 is described below
commit 84af2a46109da2b47d3b935d847b78149587af5f
Author: Josh Elser
AuthorDate: Wed Jun 12 19:15:14 2019 -0400
HBASE-22562 Remove dead code: skipControl
Signed-off-by: Reid Chan
---
.../throttle/PressureAwareCompactionThroughputController.java| 9 -
.../throttle/PressureAwareFlushThroughputController.java | 6 --
.../regionserver/throttle/PressureAwareThroughputController.java | 8
3 files changed, 23 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index 2dc5817..b24555a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -146,13 +146,4 @@ public class PressureAwareCompactionThroughputController
extends PressureAwareTh
+ throughputDesc(getMaxThroughput()) + ", activeCompactions=" +
activeOperations.size()
+ "]";
}
-
- @Override
- protected boolean skipControl(long deltaSize, long controlSize) {
-if (deltaSize < controlSize) {
- return true;
-} else {
- return false;
-}
- }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
index f301a27..ccb60ff 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
@@ -127,10 +127,4 @@ public class PressureAwareFlushThroughputController
extends PressureAwareThrough
return "DefaultFlushController [maxThroughput=" +
throughputDesc(getMaxThroughput())
+ ", activeFlushNumber=" + activeOperations.size() + "]";
}
-
- @Override
- protected boolean skipControl(long deltaSize, long controlSize) {
-// for flush, we control the flow no matter whether the flush size is small
-return false;
- }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
index 8867611..854d245 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
@@ -137,14 +137,6 @@ public abstract class PressureAwareThroughputController
extends Configured imple
return sleepTime;
}
- /**
- * Check whether to skip control given delta size and control size
- * @param deltaSize Delta size since last control
- * @param controlSize Size limit to perform control
- * @return a boolean indicates whether to skip this control
- */
- protected abstract boolean skipControl(long deltaSize, long controlSize);
-
@Override
public void finish(String opName) {
ActiveOperation operation = activeOperations.remove(opName);
[hbase] branch branch-1.4 updated: HBASE-22562 Remove dead code: skipControl
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 7ba63c8 HBASE-22562 Remove dead code: skipControl
7ba63c8 is described below
commit 7ba63c8dbd6de65df89067d08710ab3a1365b85d
Author: Josh Elser
AuthorDate: Wed Jun 12 19:15:14 2019 -0400
HBASE-22562 Remove dead code: skipControl
Signed-off-by: Reid Chan
---
.../throttle/PressureAwareCompactionThroughputController.java| 9 -
.../throttle/PressureAwareFlushThroughputController.java | 6 --
.../regionserver/throttle/PressureAwareThroughputController.java | 8
3 files changed, 23 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index c0d3b74..1681cff 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -141,13 +141,4 @@ public class PressureAwareCompactionThroughputController
extends PressureAwareTh
+ throughputDesc(getMaxThroughput()) + ", activeCompactions=" +
activeOperations.size()
+ "]";
}
-
- @Override
- protected boolean skipControl(long deltaSize, long controlSize) {
-if (deltaSize < controlSize) {
- return true;
-} else {
- return false;
-}
- }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
index f301a27..ccb60ff 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareFlushThroughputController.java
@@ -127,10 +127,4 @@ public class PressureAwareFlushThroughputController
extends PressureAwareThrough
return "DefaultFlushController [maxThroughput=" +
throughputDesc(getMaxThroughput())
+ ", activeFlushNumber=" + activeOperations.size() + "]";
}
-
- @Override
- protected boolean skipControl(long deltaSize, long controlSize) {
-// for flush, we control the flow no matter whether the flush size is small
-return false;
- }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
index 8867611..854d245 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareThroughputController.java
@@ -137,14 +137,6 @@ public abstract class PressureAwareThroughputController
extends Configured imple
return sleepTime;
}
- /**
- * Check whether to skip control given delta size and control size
- * @param deltaSize Delta size since last control
- * @param controlSize Size limit to perform control
- * @return a boolean indicates whether to skip this control
- */
- protected abstract boolean skipControl(long deltaSize, long controlSize);
-
@Override
public void finish(String opName) {
ActiveOperation operation = activeOperations.remove(opName);
[hbase] branch branch-2.1 updated: HBASE-22581 user with "CREATE" permission can grant, but not revoke permissions on created table
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 02f9c8b HBASE-22581 user with "CREATE" permission can grant, but not
revoke permissions on created table
02f9c8b is described below
commit 02f9c8b3b40e32e78b8885c43b5e0b272eceab83
Author: Istvan Toth
AuthorDate: Fri Jun 14 08:41:51 2019 +0200
HBASE-22581 user with "CREATE" permission can grant, but not revoke
permissions on created table
Signed-off-by: Reid Chan
---
.../hbase/security/access/AccessControlLists.java | 9 -
.../security/access/TestAccessController.java | 42 ++
2 files changed, 50 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index 219625b..5883120 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -259,7 +259,14 @@ public class AccessControlLists {
Delete d = new Delete(userPermissionRowKey(userPerm));
d.addColumns(ACL_LIST_FAMILY, userPermissionKey(userPerm));
try {
- t.delete(d);
+ /**
+ * We need to run the ACL delete in superuser context, to have
+ * similar authorization logic to addUserPermission().
+ * This ensures behaviour is consistent with pre 2.1.1 and 2.2+.
+ * The permission authorization has already happened here.
+ * See the TODO comment in addUserPermission for details
+ */
+ t.delete(new ArrayList<>(Arrays.asList(d)));
} finally {
t.close();
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 481e4f7..1f2724c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -3133,4 +3133,46 @@ public class TestAccessController extends SecureTestUtil
{
verifyAllowed(action, SUPERUSER);
verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER,
USER_ADMIN);
}
+
+ @Test
+ public void testTableAdmin() throws Exception {
+
+// Create a user with table admin permissions only
+User userTableAdmin = User.createUserForTesting(conf, "table_admin", new
String[0]);
+grantOnTable(TEST_UTIL, userTableAdmin.getShortName(), TEST_TABLE, null,
null,
+ Permission.Action.ADMIN);
+
+AccessTestAction grantAction = new AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+try (Connection conn = ConnectionFactory.createConnection(conf);
+Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) {
+ BlockingRpcChannel service =
acl.coprocessorService(TEST_TABLE.getName());
+ AccessControlService.BlockingInterface protocol =
+ AccessControlService.newBlockingStub(service);
+ AccessControlUtil.grant(null, protocol, USER_NONE.getShortName(),
TEST_TABLE, null, null,
+false, Action.READ);
+}
+return null;
+ }
+};
+
+AccessTestAction revokeAction = new AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+try (Connection conn = ConnectionFactory.createConnection(conf);
+Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) {
+ BlockingRpcChannel service =
acl.coprocessorService(TEST_TABLE.getName());
+ AccessControlService.BlockingInterface protocol =
+ AccessControlService.newBlockingStub(service);
+ AccessControlUtil.revoke(null, protocol, USER_NONE.getShortName(),
TEST_TABLE, null, null,
+Action.READ);
+}
+return null;
+ }
+};
+
+verifyAllowed(userTableAdmin, grantAction);
+verifyAllowed(userTableAdmin, revokeAction);
+ }
}
[hbase] branch branch-1 updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new be9f0dd HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
be9f0dd is described below
commit be9f0dd58833827a7ce038f66343dd00236dc559
Author: Reid Chan
AuthorDate: Thu Jun 20 10:49:11 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index ab76e90..573a64a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -341,6 +341,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -548,6 +553,8 @@ public class HRegionServer extends HasThread implements
this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -911,8 +918,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemstoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch branch-1.4 updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 0bc7be7 HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
0bc7be7 is described below
commit 0bc7be7ab4bb519901990779b1987536711bea3b
Author: Reid Chan
AuthorDate: Thu Jun 20 10:49:11 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f26661b..af0c680 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -330,6 +330,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -537,6 +542,8 @@ public class HRegionServer extends HasThread implements
this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -900,8 +907,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemstoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch master updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new c7c6140 HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
c7c6140 is described below
commit c7c6140396528f8f9d4dff43035a516e7ba2f22a
Author: Reid Chan
AuthorDate: Tue Jun 18 11:09:34 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 401c1b2..157f186 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -357,6 +357,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -576,6 +581,8 @@ public class HRegionServer extends HasThread implements
this.numRetries =
this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+ this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+ this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 *
1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -2018,8 +2025,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemStoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemStoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch branch-2 updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new d30a7ca HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
d30a7ca is described below
commit d30a7ca75804129486d6236d26e228e33e226baa
Author: Reid Chan
AuthorDate: Tue Jun 18 11:09:34 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index cd94c78..786ff11 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -357,6 +357,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -577,6 +582,8 @@ public class HRegionServer extends HasThread implements
this.numRetries =
this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+ this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+ this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 *
1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -2010,8 +2017,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemStoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemStoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch branch-2.2 updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 59fda38 HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
59fda38 is described below
commit 59fda3890c0368eb8f99b030963f0be6665d48eb
Author: Reid Chan
AuthorDate: Tue Jun 18 11:09:34 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 57f33cd..8ecd283 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -355,6 +355,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -572,6 +577,8 @@ public class HRegionServer extends HasThread implements
this.numRetries =
this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+ this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+ this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 *
1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -2018,8 +2025,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemStoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemStoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch branch-2.1 updated: HBASE-22596 [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 060f52d HBASE-22596 [Chore] Separate the execution period between
CompactionChecker and PeriodicMemStoreFlusher
060f52d is described below
commit 060f52de5abc915d32d1f96bd5c9531d31fc4a2b
Author: Reid Chan
AuthorDate: Tue Jun 18 11:09:34 2019 +0800
HBASE-22596 [Chore] Separate the execution period between CompactionChecker
and PeriodicMemStoreFlusher
Signed-off-by: Zach York
Signed-off-by: Xu Cang
---
.../org/apache/hadoop/hbase/regionserver/HRegionServer.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 446a19b..ef470f0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -342,6 +342,11 @@ public class HRegionServer extends HasThread implements
protected final int threadWakeFrequency;
protected final int msgInterval;
+ private static final String PERIOD_COMPACTION =
"hbase.regionserver.compaction.check.period";
+ private final int compactionCheckFrequency;
+ private static final String PERIOD_FLUSH =
"hbase.regionserver.flush.check.period";
+ private final int flushCheckFrequency;
+
protected final int numRegionsToReport;
// Stub to do region server status calls against the master.
@@ -556,6 +561,8 @@ public class HRegionServer extends HasThread implements
this.numRetries =
this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000);
+ this.compactionCheckFrequency = conf.getInt(PERIOD_COMPACTION,
this.threadWakeFrequency);
+ this.flushCheckFrequency = conf.getInt(PERIOD_FLUSH,
this.threadWakeFrequency);
this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 *
1000);
this.sleeper = new Sleeper(this.msgInterval, this);
@@ -1995,8 +2002,8 @@ public class HRegionServer extends HasThread implements
// Background thread to check for compactions; needed if region has not
gotten updates
// in a while. It will take care of not checking too frequently on
store-by-store basis.
-this.compactionChecker = new CompactionChecker(this,
this.threadWakeFrequency, this);
-this.periodicFlusher = new
PeriodicMemStoreFlusher(this.threadWakeFrequency, this);
+this.compactionChecker = new CompactionChecker(this,
this.compactionCheckFrequency, this);
+this.periodicFlusher = new
PeriodicMemStoreFlusher(this.flushCheckFrequency, this);
this.leases = new Leases(this.threadWakeFrequency);
// Create the thread to clean the moved regions list
[hbase] branch branch-1 updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated (#358)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new ebbb0e2 HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated (#358)
ebbb0e2 is described below
commit ebbb0e29873007de72befb5f025635ba8b85bb3d
Author: Reid Chan
AuthorDate: Sun Jul 7 23:15:10 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated (#358)
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index f6035c1..555b5d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -93,7 +93,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -117,7 +117,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch branch-1.4 updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated (#358)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 0f02038 HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated (#358)
0f02038 is described below
commit 0f02038a139f7164d4dd4ae31a9ef4240ca8d033
Author: Reid Chan
AuthorDate: Sun Jul 7 23:15:10 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated (#358)
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index f6035c1..555b5d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -93,7 +93,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -117,7 +117,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch master updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 605f8a1 HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated
605f8a1 is described below
commit 605f8a15bb7dabb23f1d397ca28ca0696a390497
Author: Reid Chan
AuthorDate: Fri Jul 5 13:51:04 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index 21534ce..56135df 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -101,7 +101,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -125,7 +125,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch branch-2 updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 8222631 HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated
8222631 is described below
commit 8222631ee30b2d8859086d52873be4262e31f287
Author: Reid Chan
AuthorDate: Fri Jul 5 13:51:04 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index e6f65e7..3396549 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -98,7 +98,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -122,7 +122,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch branch-2.2 updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 5a2cc9c HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated
5a2cc9c is described below
commit 5a2cc9cfc73cf3fc67ddb1e6ffb4f58789dddb3d
Author: Reid Chan
AuthorDate: Fri Jul 5 13:51:04 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index e6f65e7..3396549 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -98,7 +98,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -122,7 +122,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch branch-2.1 updated: HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never updated
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 1fb3fa2 HBASE-22656 [Metrics] Table metrics 'BatchPut' and
'BatchDelete' are never updated
1fb3fa2 is described below
commit 1fb3fa22fbdb0e8c491b3f507ca21a5188bb610d
Author: Reid Chan
AuthorDate: Fri Jul 5 13:51:04 2019 +0800
HBASE-22656 [Metrics] Table metrics 'BatchPut' and 'BatchDelete' are never
updated
Signed-off-by: Michael Stack
---
.../org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
index df50fa8..9a73bd1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
@@ -93,7 +93,7 @@ public class MetricsRegionServer {
public void updatePutBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updatePut(tn, t);
+ tableMetrics.updatePutBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowPut();
@@ -117,7 +117,7 @@ public class MetricsRegionServer {
public void updateDeleteBatch(TableName tn, long t) {
if (tableMetrics != null && tn != null) {
- tableMetrics.updateDelete(tn, t);
+ tableMetrics.updateDeleteBatch(tn, t);
}
if (t > 1000) {
serverSource.incrSlowDelete();
[hbase] branch master updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new fe450b5 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
fe450b5 is described below
commit fe450b50c1c0060773f5e3f3b884da2cf45beadc
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 04:40:32 2019 +0200
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
* HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize.
* HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize.
Deprecated old attribute and introduced a new one
* HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize. Removed
unnecessary import
* HBASE-22610 added two import configs and removed one
Signed-off-by: Reid Chan
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4d62992..4ee4977 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,20 +78,38 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
+ /**
+ * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
++ "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch branch-2 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new e95bdf4 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
e95bdf4 is described below
commit e95bdf415cea2190083150e3f8f6cea8995550d6
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 04:40:32 2019 +0200
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
* Deprecated old attribute and introduced a new one
* Removed unnecessary import
* Added two import configs and removed one
Signed-off-by: Reid Chan
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4d62992..4ee4977 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,20 +78,38 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
+ /**
+ * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
++ "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch branch-2.2 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 87b0040 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
87b0040 is described below
commit 87b00408c133f75bcf8ab97834d27a879e46b54a
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 10:55:09 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Co-authored-by: Reid Chan
Signed-off-by: Reid Chan
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 98b3c4f..01fb130 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -71,20 +71,38 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
+ /**
+ * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
+ + "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
LruBlockCache onHeapCache = createOnHeapCache(conf);
if (onHeapCache == null) {
return null;
[hbase] branch master updated: Revert "HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize"
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 0e34dcb Revert "HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize"
0e34dcb is described below
commit 0e34dcbf4b98140cf00945aa5e235f9c2dca6959
Author: Reid Chan
AuthorDate: Tue Jul 23 11:04:13 2019 +0800
Revert "HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize"
Reason: Deprecated a wrong parameter.
This reverts commit fe450b50c1c0060773f5e3f3b884da2cf45beadc.
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 3 insertions(+), 21 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4ee4977..4d62992 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,38 +78,20 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
+ * TODO: this config point is completely wrong, as it's used to determine the
+ * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
- /**
- * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
- */
- @Deprecated
- static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
-
- /**
- * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
- * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
- * compatibility.
- */
- static {
-Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
- }
-
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
-if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
- LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
-+ "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
-BLOCKCACHE_BLOCKSIZE_KEY);
-}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch branch-2 updated: Revert "HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize"
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new be5e3de Revert "HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize"
be5e3de is described below
commit be5e3de8afb277733f57d18d5d13e08241e0e888
Author: Reid Chan
AuthorDate: Tue Jul 23 11:04:13 2019 +0800
Revert "HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize"
Reason: Deprecated a wrong parameter.
This reverts commit e95bdf415cea2190083150e3f8f6cea8995550d6.
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 3 insertions(+), 21 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4ee4977..4d62992 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,38 +78,20 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
+ * TODO: this config point is completely wrong, as it's used to determine the
+ * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
- /**
- * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
- */
- @Deprecated
- static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
-
- /**
- * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
- * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
- * compatibility.
- */
- static {
-Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
- }
-
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
-if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
- LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
-+ "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
-BLOCKCACHE_BLOCKSIZE_KEY);
-}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch branch-2 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new fa4466d HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
fa4466d is described below
commit fa4466d3457dc12f903c7631bbf9f5f6acce1608
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 11:17:27 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Signed-off-by: Reid Chan
Co-authored-by: Reid Chan
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4d62992..2b97320 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,20 +78,38 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
+ /**
+ * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
+ + "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch master updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 06f5c43 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
06f5c43 is described below
commit 06f5c43de340da62e765a753c10caba5465eeae2
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 11:17:27 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Signed-off-by: Reid Chan
Co-authored-by: Reid Chan
---
.../hadoop/hbase/io/hfile/BlockCacheFactory.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
index 4d62992..2b97320 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.java
@@ -78,20 +78,38 @@ public final class BlockCacheFactory {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY =
"hbase.blockcache.external.class";
+ /**
+ * @deprecated use {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}
instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
private BlockCacheFactory() {
}
public static BlockCache createBlockCache(Configuration conf) {
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
+ + "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
if (l1Cache == null) {
return null;
[hbase] branch branch-2.1 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 49fcd92 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
49fcd92 is described below
commit 49fcd92bdf46fc130f20202c3f6693e88272ab04
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 12:05:31 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Co-authored-by: Reid Chan
Signed-off-by: Reid Chan
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index a022552..499130a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -131,10 +131,8 @@ public class CacheConfig {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
@@ -145,6 +143,21 @@ public class CacheConfig {
private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
/**
+ * @deprecated use {@link CacheConfig#BLOCKCACHE_BLOCKSIZE_KEY} instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
+ /**
* Enum of all built in external block caches.
* This is used for config.
*/
@@ -646,6 +659,11 @@ public class CacheConfig {
if (GLOBAL_BLOCK_CACHE_INSTANCE != null) {
return GLOBAL_BLOCK_CACHE_INSTANCE;
}
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key {} is deprecated now, instead please use {}. In
future release "
+ + "we will remove the deprecated config.",
DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
+BLOCKCACHE_BLOCKSIZE_KEY);
+}
if (blockCacheDisabled) {
return null;
}
[hbase] branch branch-1 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 0d8f9b5 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
0d8f9b5 is described below
commit 0d8f9b515b9b422d846a70799d40aea3d5eac264
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 12:19:08 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Co-authored-by: Reid Chan
Signed-off-by: Reid Chan
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index d020eb0..87eae04 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -129,10 +129,8 @@ public class CacheConfig {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
@@ -142,6 +140,21 @@ public class CacheConfig {
private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
/**
+ * @deprecated use {@link CacheConfig#BLOCKCACHE_BLOCKSIZE_KEY} instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
+ /**
* Enum of all built in external block caches.
* This is used for config.
*/
@@ -675,6 +688,11 @@ public class CacheConfig {
public static synchronized BlockCache instantiateBlockCache(Configuration
conf) {
if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return
GLOBAL_BLOCK_CACHE_INSTANCE;
if (blockCacheDisabled) return null;
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key " + DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY +
+ " is deprecated now, instead please use " + BLOCKCACHE_BLOCKSIZE_KEY
+". "
+ + "In future release we will remove the deprecated config.");
+}
LruBlockCache l1 = getL1(conf);
// blockCacheDisabled is set as a side-effect of getL1Internal(), so check
it again after the call.
if (blockCacheDisabled) return null;
[hbase] branch branch-1.4 updated: HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 4502b55 HBASE-22610 [BucketCache] Rename
hbase.offheapcache.minblocksize
4502b55 is described below
commit 4502b55bdc863868bc452b7876c15c0ad7400c1e
Author: syedmurtazahassan
AuthorDate: Tue Jul 23 12:19:08 2019 +0800
HBASE-22610 [BucketCache] Rename hbase.offheapcache.minblocksize
Co-authored-by: Reid Chan
Signed-off-by: Reid Chan
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index d020eb0..87eae04 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -129,10 +129,8 @@ public class CacheConfig {
/**
* The target block size used by blockcache instances. Defaults to
* {@link HConstants#DEFAULT_BLOCKSIZE}.
- * TODO: this config point is completely wrong, as it's used to determine the
- * target block size of BlockCache instances. Rename.
*/
- public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+ public static final String BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.blockcache.minblocksize";
private static final String EXTERNAL_BLOCKCACHE_KEY =
"hbase.blockcache.use.external";
private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false;
@@ -142,6 +140,21 @@ public class CacheConfig {
private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true;
/**
+ * @deprecated use {@link CacheConfig#BLOCKCACHE_BLOCKSIZE_KEY} instead.
+ */
+ @Deprecated
+ static final String DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY =
"hbase.offheapcache.minblocksize";
+
+ /**
+ * The config point hbase.offheapcache.minblocksize is completely wrong,
which is replaced by
+ * {@link BlockCacheFactory#BLOCKCACHE_BLOCKSIZE_KEY}. Keep the old config
key here for backward
+ * compatibility.
+ */
+ static {
+Configuration.addDeprecation(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
BLOCKCACHE_BLOCKSIZE_KEY);
+ }
+
+ /**
* Enum of all built in external block caches.
* This is used for config.
*/
@@ -675,6 +688,11 @@ public class CacheConfig {
public static synchronized BlockCache instantiateBlockCache(Configuration
conf) {
if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return
GLOBAL_BLOCK_CACHE_INSTANCE;
if (blockCacheDisabled) return null;
+if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
+ LOG.warn("The config key " + DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY +
+ " is deprecated now, instead please use " + BLOCKCACHE_BLOCKSIZE_KEY
+". "
+ + "In future release we will remove the deprecated config.");
+}
LruBlockCache l1 = getL1(conf);
// blockCacheDisabled is set as a side-effect of getL1Internal(), so check
it again after the call.
if (blockCacheDisabled) return null;
[hbase] branch master updated: HBASE-22628 Document the custom WAL directory (hbase.wal.dir) usage
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 7ebf80f HBASE-22628 Document the custom WAL directory (hbase.wal.dir) usage 7ebf80f is described below commit 7ebf80fe1df3113fd577259536688d11a77f3d04 Author: Pankaj AuthorDate: Wed Jul 24 08:15:46 2019 +0530 HBASE-22628 Document the custom WAL directory (hbase.wal.dir) usage Signed-off-by Reid Chan --- src/main/asciidoc/_chapters/architecture.adoc | 20 1 file changed, 20 insertions(+) diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc index 3a94740..d330d85 100644 --- a/src/main/asciidoc/_chapters/architecture.adoc +++ b/src/main/asciidoc/_chapters/architecture.adoc @@ -2491,6 +2491,26 @@ For example: If source cluster FS client configurations are copied to the destin NOTE: `DefaultSourceFSConfigurationProvider` supports only `xml` type files. It loads source cluster FS client configuration only once, so if source cluster FS client configuration files are updated, every peer(s) cluster RS must be restarted to reload the configuration. +[[arch.custom.wal.dir]] +=== Custom WAL Directory +HBASE-17437 added support for specifying a WAL directory outside the HBase root directory or even in a different FileSystem since 1.3.3/2.0+. Some FileSystems (such as Amazon S3) don’t support append or consistent writes, in such scenario WAL directory needs to be configured in a different FileSystem to avoid loss of writes. + +Following configurations were added to accomplish this: +. `hbase.wal.dir` ++ +This defines where the root WAL directory is located, could be on a different FileSystem than the root directory. WAL directory can not be set to a subdirectory of the root directory. The default value of this is the root directory if unset. ++ +. `hbase.rootdir.perms` ++ +Configures FileSystem permissions to set on the root directory. This is '700' by default. ++ +. `hbase.wal.dir.perms` ++ +Configures FileSystem permissions to set on the WAL directory FileSystem. This is '700' by default. ++ + +NOTE: While migrating to custom WAL dir (outside the HBase root directory or a different FileSystem) existing WAL files must be copied manually to new WAL dir, otherwise it may lead to data loss/inconsistency as HMaster has no information about previous WAL directory. + [[arch.hdfs]] == HDFS
[hbase] branch master updated: HBASE-22628 [Addendum] Document the custom WAL directory (hbase.wal.dir) usage
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new b83d0c0 HBASE-22628 [Addendum] Document the custom WAL directory (hbase.wal.dir) usage b83d0c0 is described below commit b83d0c035ffc2d71c11d02bdf4138abf309f2984 Author: Pankaj AuthorDate: Thu Jul 25 12:18:05 2019 +0530 HBASE-22628 [Addendum] Document the custom WAL directory (hbase.wal.dir) usage Signed-off-by: Reid Chan --- src/main/asciidoc/_chapters/architecture.adoc | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc index d330d85..886ac08 100644 --- a/src/main/asciidoc/_chapters/architecture.adoc +++ b/src/main/asciidoc/_chapters/architecture.adoc @@ -2495,19 +2495,19 @@ NOTE: `DefaultSourceFSConfigurationProvider` supports only `xml` type files. It === Custom WAL Directory HBASE-17437 added support for specifying a WAL directory outside the HBase root directory or even in a different FileSystem since 1.3.3/2.0+. Some FileSystems (such as Amazon S3) don’t support append or consistent writes, in such scenario WAL directory needs to be configured in a different FileSystem to avoid loss of writes. -Following configurations were added to accomplish this: +Following configurations are added to accomplish this: + . `hbase.wal.dir` + This defines where the root WAL directory is located, could be on a different FileSystem than the root directory. WAL directory can not be set to a subdirectory of the root directory. The default value of this is the root directory if unset. -+ + . `hbase.rootdir.perms` + Configures FileSystem permissions to set on the root directory. This is '700' by default. -+ + . `hbase.wal.dir.perms` + Configures FileSystem permissions to set on the WAL directory FileSystem. This is '700' by default. -+ NOTE: While migrating to custom WAL dir (outside the HBase root directory or a different FileSystem) existing WAL files must be copied manually to new WAL dir, otherwise it may lead to data loss/inconsistency as HMaster has no information about previous WAL directory.
[hbase] branch master updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new de98fee HBASE-22702 [Log] 'Group not found for table' is chatty
de98fee is described below
commit de98fee288814c2b62666acda1fb538dd69cbc82
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 3f3e642..9709fb5 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -207,7 +207,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
if (!misplacedRegions.contains(region)) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -239,7 +239,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -278,7 +278,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : regions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -340,7 +340,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
ServerName assignedServer = region.getValue();
String groupName =
rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -379,7 +379,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
try {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
[hbase] branch branch-2 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new cab9211 HBASE-22702 [Log] 'Group not found for table' is chatty
cab9211 is described below
commit cab92111e4f2713295a70fecd5da1e6513366ad1
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 45f5e8f..31cb7ce 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -203,7 +203,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
if (!misplacedRegions.contains(region)) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -235,7 +235,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -274,7 +274,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : regions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -336,7 +336,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
ServerName assignedServer = region.getValue();
String groupName =
rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -385,7 +385,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
try {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
[hbase] branch branch-2.2 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new a899c54 HBASE-22702 [Log] 'Group not found for table' is chatty
a899c54 is described below
commit a899c54cbc1344ba8ee0b14088d3562b9362c002
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 45f5e8f..31cb7ce 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -203,7 +203,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
if (!misplacedRegions.contains(region)) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -235,7 +235,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -274,7 +274,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : regions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -336,7 +336,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
ServerName assignedServer = region.getValue();
String groupName =
rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -385,7 +385,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
try {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
[hbase] branch branch-2.1 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 141ccc4 HBASE-22702 [Log] 'Group not found for table' is chatty
141ccc4 is described below
commit 141ccc4fd2062274f3fdb8c9fec59c5343fcec06
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 8b40d25..60296dc 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -197,7 +197,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
if (!misplacedRegions.contains(region)) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -232,7 +232,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -277,7 +277,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : regions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -339,7 +339,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
ServerName assignedServer = region.getValue();
String groupName =
rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -389,7 +389,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
try {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
[hbase] branch branch-2.0 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new e3e1c3f HBASE-22702 [Log] 'Group not found for table' is chatty
e3e1c3f is described below
commit e3e1c3fad6986fe5dd720853959e59eb5e224f7b
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 2324018..28b9e75 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -191,7 +191,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
if (!misplacedRegions.contains(region)) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -226,7 +226,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -271,7 +271,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
for (RegionInfo region : regions) {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -333,7 +333,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
ServerName assignedServer = region.getValue();
String groupName =
rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
@@ -383,7 +383,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer {
try {
String groupName =
rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
[hbase] branch branch-1.4 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 620ed11 HBASE-22702 [Log] 'Group not found for table' is chatty
620ed11 is described below
commit 620ed116a4eb4d66fd727069e01758cdf7157ea7
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 5e51128..f1b5fde 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -207,7 +207,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
if (!misplacedRegions.contains(region)) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -232,7 +232,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : misplacedRegions) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
@@ -283,7 +283,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : regions) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -346,7 +346,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
ServerName assignedServer = region.getValue();
String groupName = infoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
@@ -384,7 +384,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
try {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
info = infoManager.getRSGroup(groupName);
[hbase] branch branch-1 updated: HBASE-22702 [Log] 'Group not found for table' is chatty
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new cbb9614 HBASE-22702 [Log] 'Group not found for table' is chatty
cbb9614 is described below
commit cbb96144333efb8e855a50826252231955b8ee06
Author: syedmurtazahassan
AuthorDate: Thu Jul 25 17:13:40 2019 +0200
HBASE-22702 [Log] 'Group not found for table' is chatty
Signed-off-by Reid Chan
---
.../apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 5e51128..f1b5fde 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -207,7 +207,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
if (!misplacedRegions.contains(region)) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
@@ -232,7 +232,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : misplacedRegions) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
@@ -283,7 +283,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : regions) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
- LOG.info("Group not found for table " + region.getTable() + ", using
default");
+ LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
@@ -346,7 +346,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
ServerName assignedServer = region.getValue();
String groupName = infoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + regionInfo.getTable() + ",
using default");
+LOG.debug("Group not found for table " + regionInfo.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
@@ -384,7 +384,7 @@ public class RSGroupBasedLoadBalancer implements
RSGroupableBalancer, LoadBalanc
try {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
-LOG.info("Group not found for table " + region.getTable() + ",
using default");
+LOG.debug("Group not found for table " + region.getTable() + ",
using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
info = infoManager.getRSGroup(groupName);
[hbase] branch branch-1 updated: HBASE-22658 region_mover.rb should choose same rsgroup servers as target servers
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 1930856 HBASE-22658 region_mover.rb should choose same rsgroup
servers as target servers
1930856 is described below
commit 1930856da6fb613715616aa37de75ed0186be3c8
Author: liang.feng
AuthorDate: Tue Jul 16 22:15:40 2019 +0800
HBASE-22658 region_mover.rb should choose same rsgroup servers as target
servers
Co-authored-by: Reid Chan
Signed-off-by: stack
Signed-off-by: Andrew Purtell
Signed-off-by: Reid Chan
---
bin/region_mover.rb | 38 ++
1 file changed, 38 insertions(+)
diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 03b7c01..3e4f020 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -41,6 +41,10 @@ import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.protobuf.ProtobufUtil
import org.apache.hadoop.hbase.ServerName
import org.apache.hadoop.hbase.HRegionInfo
+import org.apache.hadoop.hbase.rsgroup.RSGroupAdmin
+import org.apache.hadoop.hbase.rsgroup.RSGroupAdminClient
+import org.apache.hadoop.hbase.client.ConnectionFactory
+import org.apache.hadoop.hbase.net.Address
# Name of this script
NAME = "region_mover"
@@ -299,6 +303,17 @@ def unloadRegions(options, hostname, port)
# Get an admin instance
admin = HBaseAdmin.new(config)
servers = getServers(admin)
+ # If rsgroup enable, get servers belongs to the same rsgroup as given server
+ if isEnableRSGroup(admin)
+$LOG.info("RegionServer group is enabled.")
+begin
+ conn = ConnectionFactory.createConnection(config)
+ rsgroupAdmin = RSGroupAdminClient.new(conn)
+ servers = getSameRSGroupServers(servers, rsgroupAdmin, hostname, port)
+ensure
+ conn.close()
+end
+ end
# Remove the server we are unloading from from list of servers.
# Side-effect is the servername that matches this hostname
servername = stripServer(servers, hostname, port)
@@ -432,6 +447,29 @@ def getFilename(options, targetServer, port)
return filename
end
+# Get servers in the same regionserver group as the given server
+def getSameRSGroupServers(servers, rsgroupAdmin, hostname, port)
+ results = []
+ rsgroup = rsgroupAdmin.getRSGroupOfServer(Address.fromParts(hostname,
+java.lang.Integer.parseInt(port)))
+ # rsgroup must be default or others, can't be nil
+ $LOG.info("Getting servers list from group: " + rsgroup.getName())
+ rsservers = rsgroup.getServers()
+ servers.each do |server|
+servername = ServerName.parseServerName(server)
+tmp = Address.fromParts(servername.getHostname(), servername.getPort())
+if rsservers.contains(tmp)
+ results << servername.getServerName()
+end
+ end
+ return results
+end
+
+# Determine whether rsgroup has been enabled
+def isEnableRSGroup(admin)
+ coprocessors = java.util.Arrays.asList(admin.getMasterCoprocessors());
+ return coprocessors.contains("RSGroupAdminEndpoint")
+end
# Do command-line parsing
options = {}
[hbase] branch branch-1.4 updated: HBASE-22658 region_mover.rb should choose same rsgroup servers as target servers
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 3c4d591 HBASE-22658 region_mover.rb should choose same rsgroup
servers as target servers
3c4d591 is described below
commit 3c4d5911a1cf127056d6c695fe787d8e7eb071a6
Author: liang.feng
AuthorDate: Tue Jul 16 22:15:40 2019 +0800
HBASE-22658 region_mover.rb should choose same rsgroup servers as target
servers
Co-authored-by: Reid Chan
Signed-off-by: stack
Signed-off-by: Andrew Purtell
Signed-off-by: Reid Chan
---
bin/region_mover.rb | 38 ++
1 file changed, 38 insertions(+)
diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 03b7c01..3e4f020 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -41,6 +41,10 @@ import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.protobuf.ProtobufUtil
import org.apache.hadoop.hbase.ServerName
import org.apache.hadoop.hbase.HRegionInfo
+import org.apache.hadoop.hbase.rsgroup.RSGroupAdmin
+import org.apache.hadoop.hbase.rsgroup.RSGroupAdminClient
+import org.apache.hadoop.hbase.client.ConnectionFactory
+import org.apache.hadoop.hbase.net.Address
# Name of this script
NAME = "region_mover"
@@ -299,6 +303,17 @@ def unloadRegions(options, hostname, port)
# Get an admin instance
admin = HBaseAdmin.new(config)
servers = getServers(admin)
+ # If rsgroup enable, get servers belongs to the same rsgroup as given server
+ if isEnableRSGroup(admin)
+$LOG.info("RegionServer group is enabled.")
+begin
+ conn = ConnectionFactory.createConnection(config)
+ rsgroupAdmin = RSGroupAdminClient.new(conn)
+ servers = getSameRSGroupServers(servers, rsgroupAdmin, hostname, port)
+ensure
+ conn.close()
+end
+ end
# Remove the server we are unloading from from list of servers.
# Side-effect is the servername that matches this hostname
servername = stripServer(servers, hostname, port)
@@ -432,6 +447,29 @@ def getFilename(options, targetServer, port)
return filename
end
+# Get servers in the same regionserver group as the given server
+def getSameRSGroupServers(servers, rsgroupAdmin, hostname, port)
+ results = []
+ rsgroup = rsgroupAdmin.getRSGroupOfServer(Address.fromParts(hostname,
+java.lang.Integer.parseInt(port)))
+ # rsgroup must be default or others, can't be nil
+ $LOG.info("Getting servers list from group: " + rsgroup.getName())
+ rsservers = rsgroup.getServers()
+ servers.each do |server|
+servername = ServerName.parseServerName(server)
+tmp = Address.fromParts(servername.getHostname(), servername.getPort())
+if rsservers.contains(tmp)
+ results << servername.getServerName()
+end
+ end
+ return results
+end
+
+# Determine whether rsgroup has been enabled
+def isEnableRSGroup(admin)
+ coprocessors = java.util.Arrays.asList(admin.getMasterCoprocessors());
+ return coprocessors.contains("RSGroupAdminEndpoint")
+end
# Do command-line parsing
options = {}
[hbase] branch master updated: HBASE-22609 [Docs] More detail documentation about 'hbase.server.thread.wakefrequency'
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a62fdcc HBASE-22609 [Docs] More detail documentation about
'hbase.server.thread.wakefrequency'
a62fdcc is described below
commit a62fdccd3b66ff2740d1a034a84f52796baf4b8b
Author: Reid Chan
AuthorDate: Mon Aug 5 14:51:00 2019 +0800
HBASE-22609 [Docs] More detail documentation about
'hbase.server.thread.wakefrequency'
Signed-off-by: stack
---
hbase-common/src/main/resources/hbase-default.xml | 26 +++
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml
b/hbase-common/src/main/resources/hbase-default.xml
index d9f5854..877cd74 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -606,8 +606,7 @@ possible configurations would overwhelm and obscure the
important.
Then the cluster's availability is at least 99% when
balancing.
-hbase.balancer.period
-
+hbase.balancer.period
30
Period at which the region balancer runs in the
Master.
@@ -631,8 +630,27 @@ possible configurations would overwhelm and obscure the
important.
hbase.server.thread.wakefrequency
1
-Time to sleep in between searches for work (in milliseconds).
-Used as sleep interval by service threads such as log roller.
+In master side, this config is the period used for FS related
behaviors:
+ checking if hdfs is out of safe mode, setting or checking hbase.version
file,
+ setting or checking hbase.id file. Using default value should be fine.
+ In regionserver side, this config is used in several places: flushing
check interval,
+ compaction check interval, wal rolling check interval. Specially, admin
can tune
+ flushing and compaction check interval by
hbase.regionserver.flush.check.period
+ and hbase.regionserver.compaction.check.period. (in
milliseconds)
+
+
+hbase.regionserver.flush.check.period
+${hbase.server.thread.wakefrequency}
+It determines the flushing check period of PeriodicFlusher in
regionserver.
+ If unset, it uses hbase.server.thread.wakefrequency as default value.
+ (in milliseconds)
+
+
+hbase.regionserver.compaction.check.period
+${hbase.server.thread.wakefrequency}
+It determines the compaction check period of
CompactionChecker in regionserver.
+ If unset, it uses hbase.server.thread.wakefrequency as default value.
+ (in milliseconds)
hbase.server.versionfile.writeattempts
[hbase] branch branch-1 updated: HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new f887207 HBASE-22774 [WAL] RegionGroupingStrategy loses its function
after split
f887207 is described below
commit f887207322ee4907c46580b628cb828e3f5539c9
Author: Reid Chan
AuthorDate: Wed Aug 14 10:27:55 2019 +0800
HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
Signed-off-by: Peter Somogyi
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 6 +-
.../hadoop/hbase/wal/BoundedGroupingStrategy.java | 2 +-
.../hbase/wal/NamespaceGroupingStrategy.java | 7 +-
.../hadoop/hbase/wal/RegionGroupingProvider.java | 3 +-
.../hbase/regionserver/TestSplitTransaction.java | 306 -
5 files changed, 306 insertions(+), 18 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c1c101a..b137c97 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7180,9 +7180,11 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
// Move the files from the temporary .splits to the final /table/region
directory
fs.commitDaughterRegion(hri);
+// rsServices can be null in UT
+WAL daughterWAL = rsServices == null ? getWAL() :rsServices.getWAL(hri);
// Create the daughter HRegion instance
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
fs.getFileSystem(),
-this.getBaseConf(), hri, this.getTableDesc(), rsServices);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), daughterWAL,
+ fs.getFileSystem(), this.getBaseConf(), hri, this.getTableDesc(),
rsServices);
r.readRequestsCount.set(this.getReadRequestsCount() / 2);
r.writeRequestsCount.set(this.getWriteRequestsCount() / 2);
return r;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
index 06f8792..b3366c2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
@@ -72,7 +72,7 @@ public class BoundedGroupingStrategy implements
RegionGroupingStrategy{
int regionGroupNumber = config.getInt(NUM_REGION_GROUPS,
DEFAULT_NUM_REGION_GROUPS);
groupNames = new String[regionGroupNumber];
for (int i = 0; i < regionGroupNumber; i++) {
- groupNames[i] = providerId + GROUP_NAME_DELIMITER + "regiongroup-" + i;
+ groupNames[i] = "regiongroup-" + i;
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
index 6193592..7b100cd 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.hbase.wal.RegionGroupingProvider.RegionGroupingStrategy
*/
@InterfaceAudience.Private
public class NamespaceGroupingStrategy implements RegionGroupingStrategy {
- private String providerId;
@Override
public String group(byte[] identifier, byte[] namespace) {
@@ -41,12 +40,10 @@ public class NamespaceGroupingStrategy implements
RegionGroupingStrategy {
} else {
namespaceString = Bytes.toString(namespace);
}
-return providerId + GROUP_NAME_DELIMITER + namespaceString;
+return namespaceString;
}
@Override
- public void init(Configuration config, String providerId) {
-this.providerId = providerId;
- }
+ public void init(Configuration config, String providerId) {}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
index b853c5b..96cef6f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
@@ -62,7 +62,6 @@ public class RegionGroupingProvider implements WALProvider {
* Map identifiers to a group number.
*/
public static interface RegionGroupingStrategy {
-String GROUP_NAME_DELIMITER = ".";
/**
* Given an identifier and a namespace, pick a group.
@@ -252,7 +251,7 @@ public class RegionGroupingProvider implements WALProvider {
publ
[hbase] branch branch-1.4 updated: HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new ae3c106 HBASE-22774 [WAL] RegionGroupingStrategy loses its function
after split
ae3c106 is described below
commit ae3c106131bca3f1b25d8e53e1409872a104a2a1
Author: Reid Chan
AuthorDate: Wed Aug 14 10:27:55 2019 +0800
HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
Signed-off-by: Peter Somogyi
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 6 +-
.../hadoop/hbase/wal/BoundedGroupingStrategy.java | 2 +-
.../hbase/wal/NamespaceGroupingStrategy.java | 7 +-
.../hadoop/hbase/wal/RegionGroupingProvider.java | 3 +-
.../hbase/regionserver/TestSplitTransaction.java | 306 -
5 files changed, 306 insertions(+), 18 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a458f39..9ccb677 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7169,9 +7169,11 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
// Move the files from the temporary .splits to the final /table/region
directory
fs.commitDaughterRegion(hri);
+// rsServices can be null in UT
+WAL daughterWAL = rsServices == null ? getWAL() :rsServices.getWAL(hri);
// Create the daughter HRegion instance
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
fs.getFileSystem(),
-this.getBaseConf(), hri, this.getTableDesc(), rsServices);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), daughterWAL,
+ fs.getFileSystem(), this.getBaseConf(), hri, this.getTableDesc(),
rsServices);
r.readRequestsCount.set(this.getReadRequestsCount() / 2);
r.writeRequestsCount.set(this.getWriteRequestsCount() / 2);
return r;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
index 06f8792..b3366c2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
@@ -72,7 +72,7 @@ public class BoundedGroupingStrategy implements
RegionGroupingStrategy{
int regionGroupNumber = config.getInt(NUM_REGION_GROUPS,
DEFAULT_NUM_REGION_GROUPS);
groupNames = new String[regionGroupNumber];
for (int i = 0; i < regionGroupNumber; i++) {
- groupNames[i] = providerId + GROUP_NAME_DELIMITER + "regiongroup-" + i;
+ groupNames[i] = "regiongroup-" + i;
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
index 6193592..7b100cd 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.hbase.wal.RegionGroupingProvider.RegionGroupingStrategy
*/
@InterfaceAudience.Private
public class NamespaceGroupingStrategy implements RegionGroupingStrategy {
- private String providerId;
@Override
public String group(byte[] identifier, byte[] namespace) {
@@ -41,12 +40,10 @@ public class NamespaceGroupingStrategy implements
RegionGroupingStrategy {
} else {
namespaceString = Bytes.toString(namespace);
}
-return providerId + GROUP_NAME_DELIMITER + namespaceString;
+return namespaceString;
}
@Override
- public void init(Configuration config, String providerId) {
-this.providerId = providerId;
- }
+ public void init(Configuration config, String providerId) {}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
index b853c5b..96cef6f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
@@ -62,7 +62,6 @@ public class RegionGroupingProvider implements WALProvider {
* Map identifiers to a group number.
*/
public static interface RegionGroupingStrategy {
-String GROUP_NAME_DELIMITER = ".";
/**
* Given an identifier and a namespace, pick a group.
@@ -252,7 +251,7 @@ public class RegionGroupingProvider implements WALProvider {
[hbase] branch branch-1.3 updated: HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new d2b310b HBASE-22774 [WAL] RegionGroupingStrategy loses its function
after split
d2b310b is described below
commit d2b310bca8140515b436aa34b2bda5c8d0236803
Author: Reid Chan
AuthorDate: Wed Aug 14 10:27:55 2019 +0800
HBASE-22774 [WAL] RegionGroupingStrategy loses its function after split
Signed-off-by: Peter Somogyi
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
---
.../apache/hadoop/hbase/regionserver/HRegion.java | 6 +-
.../hadoop/hbase/wal/BoundedGroupingStrategy.java | 2 +-
.../hbase/wal/NamespaceGroupingStrategy.java | 7 +-
.../hadoop/hbase/wal/RegionGroupingProvider.java | 3 +-
.../hbase/regionserver/TestSplitTransaction.java | 304 -
5 files changed, 303 insertions(+), 19 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 33e953f..c988863 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -6996,9 +6996,11 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
// Move the files from the temporary .splits to the final /table/region
directory
fs.commitDaughterRegion(hri);
+// rsServices can be null in UT
+WAL daughterWAL = rsServices == null ? getWAL() :rsServices.getWAL(hri);
// Create the daughter HRegion instance
-HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
fs.getFileSystem(),
-this.getBaseConf(), hri, this.getTableDesc(), rsServices);
+HRegion r = HRegion.newHRegion(this.fs.getTableDir(), daughterWAL,
+ fs.getFileSystem(), this.getBaseConf(), hri, this.getTableDesc(),
rsServices);
r.readRequestsCount.set(this.getReadRequestsCount() / 2);
r.writeRequestsCount.set(this.getWriteRequestsCount() / 2);
return r;
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
index 06f8792..b3366c2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
@@ -72,7 +72,7 @@ public class BoundedGroupingStrategy implements
RegionGroupingStrategy{
int regionGroupNumber = config.getInt(NUM_REGION_GROUPS,
DEFAULT_NUM_REGION_GROUPS);
groupNames = new String[regionGroupNumber];
for (int i = 0; i < regionGroupNumber; i++) {
- groupNames[i] = providerId + GROUP_NAME_DELIMITER + "regiongroup-" + i;
+ groupNames[i] = "regiongroup-" + i;
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
index 6193592..7b100cd 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NamespaceGroupingStrategy.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.hbase.wal.RegionGroupingProvider.RegionGroupingStrategy
*/
@InterfaceAudience.Private
public class NamespaceGroupingStrategy implements RegionGroupingStrategy {
- private String providerId;
@Override
public String group(byte[] identifier, byte[] namespace) {
@@ -41,12 +40,10 @@ public class NamespaceGroupingStrategy implements
RegionGroupingStrategy {
} else {
namespaceString = Bytes.toString(namespace);
}
-return providerId + GROUP_NAME_DELIMITER + namespaceString;
+return namespaceString;
}
@Override
- public void init(Configuration config, String providerId) {
-this.providerId = providerId;
- }
+ public void init(Configuration config, String providerId) {}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
index a725989..e96f4d7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
@@ -61,7 +61,6 @@ public class RegionGroupingProvider implements WALProvider {
* Map identifiers to a group number.
*/
public static interface RegionGroupingStrategy {
-String GROUP_NAME_DELIMITER = ".";
/**
* Given an identif
[hbase] branch master updated: HBASE-22873 Typo in block caching docs
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 7903f55 HBASE-22873 Typo in block caching docs 7903f55 is described below commit 7903f55c18d561d7451e91b8a014936a3116f142 Author: Shuai Lin AuthorDate: Sat Aug 17 19:10:46 2019 +0800 HBASE-22873 Typo in block caching docs Signed-off-by: Reid Chan --- src/main/asciidoc/_chapters/architecture.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc index a2fe2f1..848b8fd 100644 --- a/src/main/asciidoc/_chapters/architecture.adoc +++ b/src/main/asciidoc/_chapters/architecture.adoc @@ -837,7 +837,7 @@ Here are two use cases: Setting block caching on such a table is a waste of memory and CPU cycles, more so that it will generate more garbage to pick up by the JVM. For more information on monitoring GC, see <>. * Mapping a table: In a typical MapReduce job that takes a table in input, every row will be read only once so there's no need to put them into the block cache. - The Scan object has the option of turning this off via the setCaching method (set it to false). You can still keep block caching turned on on this table if you need fast random read access. + The Scan object has the option of turning this off via the setCacheBlocks method (set it to false). You can still keep block caching turned on on this table if you need fast random read access. An example would be counting the number of rows in a table that serves live traffic, caching every block of that table would create massive churn and would surely evict data that's currently in use. [[data.blocks.in.fscache]]
[hbase] branch branch-1 updated: HBASE-25031 [Flaky Test] TestReplicationDisableInactivePeer#testDisableInactivePeer (#2402)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 982aa33 HBASE-25031 [Flaky Test]
TestReplicationDisableInactivePeer#testDisableInactivePeer (#2402)
982aa33 is described below
commit 982aa33cda77225c29afb918b8812b441e44a80c
Author: Reid Chan
AuthorDate: Mon Sep 28 12:23:59 2020 +0800
HBASE-25031 [Flaky Test]
TestReplicationDisableInactivePeer#testDisableInactivePeer (#2402)
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/replication/TestReplicationDisableInactivePeer.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
index d73d7f8..9896b1b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
@@ -62,6 +62,7 @@ public class TestReplicationDisableInactivePeer extends
TestReplicationBase {
// disable and start the peer
admin.disablePeer("2");
utility2.startMiniHBaseCluster(1, 2);
+htable2 = utility2.getConnection().getTable(tableName);
Get get = new Get(rowkey);
for (int i = 0; i < NB_RETRIES; i++) {
Result res = htable2.get(get);
[hbase] branch branch-1 updated (982aa33 -> ccb4e89)
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a change to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 982aa33 HBASE-25031 [Flaky Test] TestReplicationDisableInactivePeer#testDisableInactivePeer (#2402) add ccb4e89 HBASE-25030 [Flaky Test] TestRestartCluster#testClusterRestart (#2401) No new revisions were added by this update. Summary of changes: .../src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java | 1 + 1 file changed, 1 insertion(+)
[hbase] branch branch-1 updated: HBASE-25025 [Flaky Test][branch-1] TestFromClientSide#testCheckAndDeleteWithCompareOp (#2396)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 93b76fd HBASE-25025 [Flaky Test][branch-1]
TestFromClientSide#testCheckAndDeleteWithCompareOp (#2396)
93b76fd is described below
commit 93b76fdb322130b474a122494a3844893e628443
Author: Reid Chan
AuthorDate: Tue Sep 29 14:38:22 2020 +0800
HBASE-25025 [Flaky Test][branch-1]
TestFromClientSide#testCheckAndDeleteWithCompareOp (#2396)
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/client/TestFromClientSide.java| 92 +-
1 file changed, 55 insertions(+), 37 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 0e715a9..4c2ec1f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -5067,73 +5067,91 @@ public class TestFromClientSide {
FAMILY);
TEST_UTIL.waitTableAvailable(TableName.valueOf("testCheckAndDeleteWithCompareOp"),
1);
-Put put2 = new Put(ROW);
-put2.add(FAMILY, QUALIFIER, value2);
-table.put(put2);
+Put = new Put(ROW);
+.add(FAMILY, QUALIFIER, value2);
-Put put3 = new Put(ROW);
-put3.add(FAMILY, QUALIFIER, value3);
+Put = new Put(ROW);
+.add(FAMILY, QUALIFIER, value3);
Delete delete = new Delete(ROW);
delete.deleteColumns(FAMILY, QUALIFIER);
// cell = "", using "" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
// turns out "match"
+table.put();
+assertTrue(verifyPut(table, , value2));
boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER,
CompareOp.GREATER, value1, delete);
-assertEquals(ok, false);
+// is less than , > || >= should be false
+assertFalse(ok);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1,
delete);
-assertEquals(ok, false);
+assertFalse(ok);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER,
CompareOp.GREATER_OR_EQUAL, value1, delete);
-assertEquals(ok, false);
+assertFalse(ok);
+// is less than , < || <= should be true
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1,
delete);
-assertEquals(ok, true);
-table.put(put2);
+assertTrue(ok);
+table.put();
+assertTrue(verifyPut(table, , value2));
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL,
value1, delete);
-assertEquals(ok, true);
-table.put(put2);
-
-assertEquals(ok, true);
+assertTrue(ok);
// cell = "", using "" to compare only
LARGER/LARGER_OR_EQUAL/NOT_EQUAL
// turns out "match"
-table.put(put3);
+table.put();
+assertTrue(verifyPut(table, , value3));
+// is larger than , < || <= shoule be false
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4,
delete);
-
-assertEquals(ok, false);
+assertFalse(ok);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL,
value4, delete);
-
-assertEquals(ok, false);
+assertFalse(ok);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4,
delete);
-
-assertEquals(ok, false);
+assertFalse(ok);
+// is larger than , (> || >= || !=) shoule be true
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER,
value4, delete);
-
-assertEquals(ok, true);
-table.put(put3);
+assertTrue(ok);
+table.put();
+assertTrue(verifyPut(table, , value3));
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER,
CompareOp.GREATER_OR_EQUAL, value4, delete);
-assertEquals(ok, true);
-table.put(put3);
+assertTrue(ok);
+table.put();
+assertTrue(verifyPut(table, , value3));
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL,
value4, delete);
-
-assertEquals(ok, true);
+assertTrue(ok);
// cell = "", using "" to compare only
GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
// turns out "match"
-table.put(put2);
+// equals to , != shoule be all false
+table.put();
+assertTrue(verifyPut(table, , value2));
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER,
value2, delete);
-assertEquals(ok, false);
+assertFalse(ok);
ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL,
value2, delete);
-assertEquals(ok, false);
+assertFalse(ok);
ok = tabl
[hbase] branch branch-1 updated: HBASE-25114 [Flake Test][branch-1] TestFromClientSide#testCacheOnWriteEvictOnClose (#2470)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new e719a5b HBASE-25114 [Flake Test][branch-1]
TestFromClientSide#testCacheOnWriteEvictOnClose (#2470)
e719a5b is described below
commit e719a5b589b3eb2fd7bff47e6332aa3d61e3b2f6
Author: Reid Chan
AuthorDate: Wed Sep 30 00:17:40 2020 +0800
HBASE-25114 [Flake Test][branch-1]
TestFromClientSide#testCacheOnWriteEvictOnClose (#2470)
Signed-off-by: Viraj Jasani
---
.../test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 4c2ec1f..c10ff07 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -5386,7 +5386,7 @@ public class TestFromClientSide {
waitForStoreFileCount(store, 1, 1); // wait 10 seconds max
assertEquals(1, store.getStorefilesCount());
// evicted two data blocks and two index blocks and compaction does not
cache new blocks
-expectedBlockCount = 0;
+expectedBlockCount -= 4;
assertEquals(expectedBlockCount, cache.getBlockCount());
expectedBlockHits += 2;
assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
[hbase] branch branch-1 updated: HBASE-25122 [Flake Test][branch-1] TestExportSnapshotWithTemporaryDirectory (#2472)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 6edeb5d HBASE-25122 [Flake Test][branch-1]
TestExportSnapshotWithTemporaryDirectory (#2472)
6edeb5d is described below
commit 6edeb5d1ecfd8b0155ff3c9022c6de2101565177
Author: Reid Chan
AuthorDate: Wed Sep 30 18:55:28 2020 +0800
HBASE-25122 [Flake Test][branch-1] TestExportSnapshotWithTemporaryDirectory
(#2472)
* Remove unused imports
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/snapshot/TestExportSnapshot.java | 6 ++
.../TestExportSnapshotWithTemporaryDirectory.java | 21 +++--
2 files changed, 9 insertions(+), 18 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index f045ada..e96a1de 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -40,8 +40,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import
org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
@@ -67,7 +65,7 @@ import org.junit.rules.TestRule;
public class TestExportSnapshot {
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
- private static final Log LOG = LogFactory.getLog(TestExportSnapshot.class);
+ protected static final Log LOG = LogFactory.getLog(TestExportSnapshot.class);
protected final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
@@ -79,7 +77,7 @@ public class TestExportSnapshot {
private TableName tableName;
private Admin admin;
- public static void setUpBaseConf(Configuration conf) {
+ public static void setUpBaseConf(Configuration conf) throws Exception {
conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
conf.setInt("hbase.regionserver.msginterval", 100);
conf.setInt("hbase.client.pause", 250);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java
index d50f262..b9c69d7 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.hbase.snapshot;
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.UUID;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -31,24 +27,21 @@ import org.junit.experimental.categories.Category;
@Category({MediumTests.class})
public class TestExportSnapshotWithTemporaryDirectory extends
TestExportSnapshot {
- protected static String TEMP_DIR = Paths.get("").toAbsolutePath().toString()
+ Path.SEPARATOR
- + UUID.randomUUID().toString();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
-setUpBaseConf(TEST_UTIL.getConfiguration());
+Configuration conf = TEST_UTIL.getConfiguration();
+TestExportSnapshot.setUpBaseConf(conf);
TEST_UTIL.startMiniCluster(3);
+Path rootDir =
TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+LOG.info("Root dir: " + rootDir);
+conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR,
+ new Path(rootDir.getParent(), ".tmpdir").toUri().toString());
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TestExportSnapshot.tearDownAfterClass();
-FileUtils.deleteDirectory(new File(TEMP_DIR));
}
- public static void setUpBaseConf(Configuration conf) {
-TestExportSnapshot.setUpBaseConf(conf);
-conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new
Path(TEMP_DIR, ".tmpdir").toUri());
- }
-}
\ No newline at end of file
+}
[hbase] branch branch-1 updated: HBASE-24849 Branch-1 Backport : HBASE-24665 MultiWAL : Avoid rolling of ALL WALs when one of the WAL needs a roll (#2194)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new e066951 HBASE-24849 Branch-1 Backport : HBASE-24665 MultiWAL : Avoid
rolling of ALL WALs when one of the WAL needs a roll (#2194)
e066951 is described below
commit e06695112a358706344cc8682f2713b43daab340
Author: WenFeiYi
AuthorDate: Fri Oct 16 20:42:18 2020 +0800
HBASE-24849 Branch-1 Backport : HBASE-24665 MultiWAL : Avoid rolling of ALL
WALs when one of the WAL needs a roll (#2194)
Signed-off-by: Reid Chan
---
.../hadoop/hbase/regionserver/LogRoller.java | 108 ++-
.../hadoop/hbase/regionserver/TestLogRoller.java | 114 +
2 files changed, 194 insertions(+), 28 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index fd208c2..08d5a33 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.util.Bytes;
@@ -56,23 +57,27 @@ public class LogRoller extends HasThread {
private static final Log LOG = LogFactory.getLog(LogRoller.class);
private final ReentrantLock rollLock = new ReentrantLock();
private final AtomicBoolean rollLog = new AtomicBoolean(false);
- private final ConcurrentHashMap walNeedsRoll =
- new ConcurrentHashMap();
+ private final ConcurrentHashMap wals =
+ new ConcurrentHashMap();
private final Server server;
protected final RegionServerServices services;
- private volatile long lastrolltime = System.currentTimeMillis();
// Period to roll log.
- private final long rollperiod;
+ private final long rollPeriod;
private final int threadWakeFrequency;
// The interval to check low replication on hlog's pipeline
- private long checkLowReplicationInterval;
+ private final long checkLowReplicationInterval;
public void addWAL(final WAL wal) {
-if (null == walNeedsRoll.putIfAbsent(wal, Boolean.FALSE)) {
+if (null == wals.putIfAbsent(wal, new RollController(wal))) {
wal.registerWALActionsListener(new WALActionsListener.Base() {
@Override
public void logRollRequested(WALActionsListener.RollRequestReason
reason) {
- walNeedsRoll.put(wal, Boolean.TRUE);
+ RollController controller = wals.get(wal);
+ if (controller == null) {
+wals.putIfAbsent(wal, new RollController(wal));
+controller = wals.get(wal);
+ }
+ controller.requestRoll();
// TODO logs will contend with each other here, replace with e.g.
DelayedQueue
synchronized(rollLog) {
rollLog.set(true);
@@ -84,8 +89,8 @@ public class LogRoller extends HasThread {
}
public void requestRollAll() {
-for (WAL wal : walNeedsRoll.keySet()) {
- walNeedsRoll.put(wal, Boolean.TRUE);
+for (RollController controller : wals.values()) {
+ controller.requestRoll();
}
synchronized(rollLog) {
rollLog.set(true);
@@ -98,7 +103,7 @@ public class LogRoller extends HasThread {
super("LogRoller");
this.server = server;
this.services = services;
-this.rollperiod = this.server.getConfiguration().
+this.rollPeriod = this.server.getConfiguration().
getLong("hbase.regionserver.logroll.period", 360);
this.threadWakeFrequency = this.server.getConfiguration().
getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
@@ -120,9 +125,9 @@ public class LogRoller extends HasThread {
*/
void checkLowReplication(long now) {
try {
- for (Entry entry : walNeedsRoll.entrySet()) {
+ for (Entry entry : wals.entrySet()) {
WAL wal = entry.getKey();
-boolean neeRollAlready = entry.getValue();
+boolean neeRollAlready = entry.getValue().needsRoll(now);
if(wal instanceof FSHLog && !neeRollAlready) {
FSHLog hlog = (FSHLog)wal;
if ((now - hlog.getLastTimeCheckLowReplication())
@@ -139,11 +144,16 @@ public class LogRoller extends HasThread {
@Override
public void run() {
while (!server.isStopped()) {
- long now = System.currentTimeMillis();
+ long now = EnvironmentEdgeManager.currentTime();
checkLo
[hbase] branch branch-1 updated: HBASE-25195 [branch-1] getNumOpenConnections is not effective (#2557)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new d3ac342 HBASE-25195 [branch-1] getNumOpenConnections is not effective
(#2557)
d3ac342 is described below
commit d3ac3420e5b9c7ba39497fc9c86e5a47c0332eb1
Author: Reid Chan
AuthorDate: Mon Oct 19 10:54:15 2020 +0800
HBASE-25195 [branch-1] getNumOpenConnections is not effective (#2557)
Signed-off-by: Viraj Jasani
---
.../org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
index 6ba7ea2..2f3745f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
@@ -65,10 +65,10 @@ public class MetricsHBaseServerWrapperImpl implements
MetricsHBaseServerWrapper
@Override
public int getNumOpenConnections() {
-if (!isServerStarted() || this.server.connectionList == null) {
+if (!isServerStarted()) {
return 0;
}
-return server.connectionList.size();
+return server.numConnections;
}
@Override
[hbase] branch master updated: HBASE-25189 [Metrics] Add checkAndPut and checkAndDelete latency metrics at table level (#2549)
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new e5d4e2f HBASE-25189 [Metrics] Add checkAndPut and checkAndDelete
latency metrics at table level (#2549)
e5d4e2f is described below
commit e5d4e2fc8138cba0c4a1da2b42b51042da3d9c7e
Author: Reid Chan
AuthorDate: Sun Oct 25 17:46:14 2020 +0800
HBASE-25189 [Metrics] Add checkAndPut and checkAndDelete latency metrics at
table level (#2549)
Signed-off-by: Viraj Jasani
---
.../hbase/regionserver/MetricsTableLatencies.java | 25 +++
.../regionserver/MetricsTableLatenciesImpl.java| 36 ++
.../hbase/regionserver/MetricsRegionServer.java| 15 +++--
.../hadoop/hbase/regionserver/RSRpcServices.java | 9 --
.../regionserver/RegionServerTableMetrics.java | 12
.../regionserver/TestMetricsRegionServer.java | 17 ++
6 files changed, 94 insertions(+), 20 deletions(-)
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatencies.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatencies.java
index 231bad1..2aeb82b 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatencies.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatencies.java
@@ -53,6 +53,9 @@ public interface MetricsTableLatencies {
String DELETE_BATCH_TIME = "deleteBatchTime";
String INCREMENT_TIME = "incrementTime";
String APPEND_TIME = "appendTime";
+ String CHECK_AND_DELETE_TIME = "checkAndDeleteTime";
+ String CHECK_AND_PUT_TIME = "checkAndPutTime";
+ String CHECK_AND_MUTATE_TIME = "checkAndMutateTime";
/**
* Update the Put time histogram
@@ -125,4 +128,26 @@ public interface MetricsTableLatencies {
* @param t time it took
*/
void updateScanTime(String tableName, long t);
+
+ /**
+ * Update the CheckAndDelete time histogram.
+ * @param nameAsString The table the metric is for
+ * @param time time it took
+ */
+ void updateCheckAndDelete(String nameAsString, long time);
+
+ /**
+ * Update the CheckAndPut time histogram.
+ * @param nameAsString The table the metric is for
+ * @param time time it took
+ */
+ void updateCheckAndPut(String nameAsString, long time);
+
+ /**
+ * Update the CheckAndMutate time histogram.
+ * @param nameAsString The table the metric is for
+ * @param time time it took
+ */
+ void updateCheckAndMutate(String nameAsString, long time);
+
}
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
index 5a3f3b9..5e13a61 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableLatenciesImpl.java
@@ -47,6 +47,9 @@ public class MetricsTableLatenciesImpl extends BaseSourceImpl
implements Metrics
final MetricHistogram deleteBatchTimeHisto;
final MetricHistogram scanTimeHisto;
final MetricHistogram scanSizeHisto;
+final MetricHistogram checkAndDeleteTimeHisto;
+final MetricHistogram checkAndPutTimeHisto;
+final MetricHistogram checkAndMutateTimeHisto;
TableHistograms(DynamicMetricsRegistry registry, TableName tn) {
getTimeHisto = registry.newTimeHistogram(qualifyMetricsName(tn,
GET_TIME));
@@ -60,6 +63,12 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
qualifyMetricsName(tn, DELETE_BATCH_TIME));
scanTimeHisto = registry.newTimeHistogram(qualifyMetricsName(tn,
SCAN_TIME));
scanSizeHisto = registry.newSizeHistogram(qualifyMetricsName(tn,
SCAN_SIZE));
+ checkAndDeleteTimeHisto =
+registry.newTimeHistogram(qualifyMetricsName(tn,
CHECK_AND_DELETE_TIME));
+ checkAndPutTimeHisto =
+registry.newTimeHistogram(qualifyMetricsName(tn, CHECK_AND_PUT_TIME));
+ checkAndMutateTimeHisto =
+registry.newTimeHistogram(qualifyMetricsName(tn,
CHECK_AND_MUTATE_TIME));
}
public void updatePut(long time) {
@@ -97,6 +106,18 @@ public class MetricsTableLatenciesImpl extends
BaseSourceImpl implements Metrics
public void updateScanTime(long t) {
scanTimeHisto.add(t);
}
+
+public void updateCheckAndDeleteTime(long t) {
+ checkAndDeleteTimeHisto.add(t);
+}
+
+public void updateCheckAndPutTime(long t) {
+ checkAndPutTimeHisto.add(t);
+}
+
+public void updateCheckAndMutateTime(long t) {
+ checkAndMutateTimeHisto.a
