hbase git commit: HBASE-17779 disable_table_replication returns misleading message and does not turn off replication (Janos Gub)
Repository: hbase Updated Branches: refs/heads/master 44b255889 -> 777fea552 HBASE-17779 disable_table_replication returns misleading message and does not turn off replication (Janos Gub) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/777fea55 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/777fea55 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/777fea55 Branch: refs/heads/master Commit: 777fea552eab3262e95053b2fc757fc49dfad96d Parents: 44b2558 Author: tedyu Authored: Tue Mar 14 12:13:34 2017 -0700 Committer: tedyu Committed: Tue Mar 14 12:13:34 2017 -0700 -- .../apache/hadoop/hbase/client/HBaseAdmin.java | 35 +++- .../TestReplicationAdminWithClusters.java | 17 ++ 2 files changed, 44 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/777fea55/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 1f143b5..6918184 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -4228,14 +4228,16 @@ public class HBaseAdmin implements Admin { /** * Set the table's replication switch if the table's replication switch is already not set. * @param tableName name of the table - * @param isRepEnabled is replication switch enable or disable + * @param enableRep is replication switch enable or disable * @throws IOException if a remote or network exception occurs */ - private void setTableRep(final TableName tableName, boolean isRepEnabled) throws IOException { + private void setTableRep(final TableName tableName, boolean enableRep) throws IOException { HTableDescriptor htd = getTableDescriptor(tableName); -if (isTableRepEnabled(htd) ^ isRepEnabled) { +ReplicationState currentReplicationState = getTableReplicationState(htd); +if (enableRep && currentReplicationState != ReplicationState.ENABLED +|| !enableRep && currentReplicationState != ReplicationState.DISABLED) { for (HColumnDescriptor hcd : htd.getFamilies()) { -hcd.setScope(isRepEnabled ? HConstants.REPLICATION_SCOPE_GLOBAL +hcd.setScope(enableRep ? HConstants.REPLICATION_SCOPE_GLOBAL : HConstants.REPLICATION_SCOPE_LOCAL); } modifyTable(tableName, htd); @@ -4243,17 +4245,34 @@ public class HBaseAdmin implements Admin { } /** + * This enum indicates the current state of the replication for a given table. + */ + private enum ReplicationState { +ENABLED, // all column families enabled +MIXED, // some column families enabled, some disabled +DISABLED // all column families disabled + } + + /** * @param htd table descriptor details for the table to check - * @return true if table's replication switch is enabled + * @return ReplicationState the current state of the table. */ - private boolean isTableRepEnabled(HTableDescriptor htd) { + private ReplicationState getTableReplicationState(HTableDescriptor htd) { +boolean hasEnabled = false; +boolean hasDisabled = false; + for (HColumnDescriptor hcd : htd.getFamilies()) { if (hcd.getScope() != HConstants.REPLICATION_SCOPE_GLOBAL && hcd.getScope() != HConstants.REPLICATION_SCOPE_SERIAL) { -return false; +hasDisabled = true; + } else { +hasEnabled = true; } } -return true; + +if (hasEnabled && hasDisabled) return ReplicationState.MIXED; +if (hasEnabled) return ReplicationState.ENABLED; +return ReplicationState.DISABLED; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/777fea55/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java index b44ecbf..312a90a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java @@ -76,6 +76,23 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase { } @Test(timeout = 30) + public void disableNotFullReplication() throws Exception
hbase git commit: HBASE-17779 disable_table_replication returns misleading message and does not turn off replication (Janos Gub)
Repository: hbase Updated Branches: refs/heads/branch-1 c97905a96 -> 318298047 HBASE-17779 disable_table_replication returns misleading message and does not turn off replication (Janos Gub) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/31829804 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/31829804 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/31829804 Branch: refs/heads/branch-1 Commit: 318298047be788143405cfb101e1b0401f9e6b7a Parents: c97905a Author: tedyu Authored: Tue Mar 14 12:34:04 2017 -0700 Committer: tedyu Committed: Tue Mar 14 12:34:04 2017 -0700 -- .../client/replication/ReplicationAdmin.java| 35 +++- .../TestReplicationAdminWithClusters.java | 17 ++ 2 files changed, 44 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/31829804/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java index f9cf6b3..80f6e10 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -647,15 +647,17 @@ public class ReplicationAdmin implements Closeable { /** * Set the table's replication switch if the table's replication switch is already not set. * @param tableName name of the table - * @param isRepEnabled is replication switch enable or disable + * @param enableRep is replication switch enable or disable * @throws IOException if a remote or network exception occurs */ - private void setTableRep(final TableName tableName, boolean isRepEnabled) throws IOException { + private void setTableRep(final TableName tableName, boolean enableRep) throws IOException { Admin admin = null; try { admin = this.connection.getAdmin(); HTableDescriptor htd = admin.getTableDescriptor(tableName); - if (isTableRepEnabled(htd) ^ isRepEnabled) { + ReplicationState currentReplicationState = getTableReplicationState(htd); + if (enableRep && currentReplicationState != ReplicationState.ENABLED + || !enableRep && currentReplicationState != ReplicationState.DISABLED) { boolean isOnlineSchemaUpdateEnabled = this.connection.getConfiguration() .getBoolean("hbase.online.schema.update.enable", true); @@ -663,7 +665,7 @@ public class ReplicationAdmin implements Closeable { admin.disableTable(tableName); } for (HColumnDescriptor hcd : htd.getFamilies()) { - hcd.setScope(isRepEnabled ? HConstants.REPLICATION_SCOPE_GLOBAL + hcd.setScope(enableRep ? HConstants.REPLICATION_SCOPE_GLOBAL : HConstants.REPLICATION_SCOPE_LOCAL); } admin.modifyTable(tableName, htd); @@ -684,17 +686,34 @@ public class ReplicationAdmin implements Closeable { } /** + * This enum indicates the current state of the replication for a given table. + */ + private enum ReplicationState { +ENABLED, // all column families enabled +MIXED, // some column families enabled, some disabled +DISABLED // all column families disabled + } + + /** * @param htd table descriptor details for the table to check - * @return true if table's replication switch is enabled + * @return ReplicationState the current state of the table. */ - private boolean isTableRepEnabled(HTableDescriptor htd) { + private ReplicationState getTableReplicationState(HTableDescriptor htd) { +boolean hasEnabled = false; +boolean hasDisabled = false; + for (HColumnDescriptor hcd : htd.getFamilies()) { if (hcd.getScope() != HConstants.REPLICATION_SCOPE_GLOBAL && hcd.getScope() != HConstants.REPLICATION_SCOPE_SERIAL) { -return false; +hasDisabled = true; + } else { +hasEnabled = true; } } -return true; + +if (hasEnabled && hasDisabled) return ReplicationState.MIXED; +if (hasEnabled) return ReplicationState.ENABLED; +return ReplicationState.DISABLED; } private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig) http://git-wip-us.apache.org/repos/asf/hbase/blob/31829804/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestRe
[5/5] hbase git commit: HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable
HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3ced5e4c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3ced5e4c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3ced5e4c Branch: refs/heads/branch-1.1 Commit: 3ced5e4c5e64b18519aa99d950ac2c0599e49b1f Parents: ab9071f Author: Andrew Purtell Authored: Tue Mar 14 13:23:11 2017 -0700 Committer: Andrew Purtell Committed: Tue Mar 14 13:24:06 2017 -0700 -- .../java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3ced5e4c/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index 37b7bd3..b4394a4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -121,7 +121,9 @@ public class BoundedByteBufferPool { lock.unlock(); } if (!success) { - LOG.warn("At capacity: " + this.buffers.size()); + if (LOG.isDebugEnabled()) { +LOG.debug("At capacity: " + this.buffers.size()); + } } else { if (average > this.runningAverage && average < this.maxByteBufferSizeToCache) { this.runningAverage = average;
[1/5] hbase git commit: HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable
Repository: hbase Updated Branches: refs/heads/branch-1 318298047 -> d7666b606 refs/heads/branch-1.1 ab9071fa6 -> 3ced5e4c5 refs/heads/branch-1.2 f61f02b2b -> 17ef9fb15 refs/heads/branch-1.3 834488d43 -> 48cbce7ce refs/heads/master 777fea552 -> 14fb57cab HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/14fb57ca Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/14fb57ca Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/14fb57ca Branch: refs/heads/master Commit: 14fb57cab2fd8c0117d59669018b09e29bd6e387 Parents: 777fea5 Author: Andrew Purtell Authored: Tue Mar 14 13:23:11 2017 -0700 Committer: Andrew Purtell Committed: Tue Mar 14 13:23:11 2017 -0700 -- .../java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/14fb57ca/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index 079a277..7bce0e5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -155,8 +155,8 @@ public class BoundedByteBufferPool { long prevState = stateRef.get(); countOfBuffers = toCountOfBuffers(prevState); if (countOfBuffers >= maxToCache) { -if (LOG.isWarnEnabled()) { - LOG.warn("At capacity: " + countOfBuffers); +if (LOG.isDebugEnabled()) { + LOG.debug("At capacity: " + countOfBuffers); } return; }
[2/5] hbase git commit: HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable
HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d7666b60 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d7666b60 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d7666b60 Branch: refs/heads/branch-1 Commit: d7666b60662cfc8492e396565b29494407c7843c Parents: 3182980 Author: Andrew Purtell Authored: Tue Mar 14 13:23:11 2017 -0700 Committer: Andrew Purtell Committed: Tue Mar 14 13:23:47 2017 -0700 -- .../java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d7666b60/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index acd7a18..4d52614 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -126,7 +126,9 @@ public class BoundedByteBufferPool { lock.unlock(); } if (!success) { - LOG.warn("At capacity: " + this.buffers.size()); + if (LOG.isDebugEnabled()) { +LOG.debug("At capacity: " + this.buffers.size()); + } } else { if (average > this.runningAverage && average < this.maxByteBufferSizeToCache) { this.runningAverage = average;
[4/5] hbase git commit: HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable
HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/17ef9fb1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/17ef9fb1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/17ef9fb1 Branch: refs/heads/branch-1.2 Commit: 17ef9fb15cca5f4075e90b78fdbcee7d0f80e238 Parents: f61f02b Author: Andrew Purtell Authored: Tue Mar 14 13:23:11 2017 -0700 Committer: Andrew Purtell Committed: Tue Mar 14 13:24:03 2017 -0700 -- .../java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/17ef9fb1/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index c685a92..07456d1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -121,7 +121,9 @@ public class BoundedByteBufferPool { lock.unlock(); } if (!success) { - LOG.warn("At capacity: " + this.buffers.size()); + if (LOG.isDebugEnabled()) { +LOG.debug("At capacity: " + this.buffers.size()); + } } else { if (average > this.runningAverage && average < this.maxByteBufferSizeToCache) { this.runningAverage = average;
[3/5] hbase git commit: HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable
HBASE-17780 BoundedByteBufferPool "At capacity" messages are not actionable Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/48cbce7c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/48cbce7c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/48cbce7c Branch: refs/heads/branch-1.3 Commit: 48cbce7ce07a795786d2240f55577a7aeaf041cc Parents: 834488d Author: Andrew Purtell Authored: Tue Mar 14 13:23:11 2017 -0700 Committer: Andrew Purtell Committed: Tue Mar 14 13:24:00 2017 -0700 -- .../java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/48cbce7c/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index acd7a18..4d52614 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -126,7 +126,9 @@ public class BoundedByteBufferPool { lock.unlock(); } if (!success) { - LOG.warn("At capacity: " + this.buffers.size()); + if (LOG.isDebugEnabled()) { +LOG.debug("At capacity: " + this.buffers.size()); + } } else { if (average > this.runningAverage && average < this.maxByteBufferSizeToCache) { this.runningAverage = average;
hbase git commit: HBASE-17768 [C++] Makefile should recompile only the changed sources (Sudeep Sunthankar)
Repository: hbase Updated Branches: refs/heads/HBASE-14850 9b43493f8 -> d2bc6b1b7 HBASE-17768 [C++] Makefile should recompile only the changed sources (Sudeep Sunthankar) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2bc6b1b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2bc6b1b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2bc6b1b Branch: refs/heads/HBASE-14850 Commit: d2bc6b1b78efe87ff540ccfa8374ee5daec13da2 Parents: 9b43493 Author: Enis Soztutar Authored: Tue Mar 14 14:35:22 2017 -0700 Committer: Enis Soztutar Committed: Tue Mar 14 14:35:22 2017 -0700 -- hbase-native-client/Makefile| 132 +-- hbase-native-client/Makefile.protos | 68 +--- 2 files changed, 113 insertions(+), 87 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d2bc6b1b/hbase-native-client/Makefile -- diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile index 166c6ab..143c00c 100644 --- a/hbase-native-client/Makefile +++ b/hbase-native-client/Makefile @@ -15,82 +15,94 @@ # specific language governing permissions and limitations # under the License. -#use "gcc" to compile source files -CC:=g++ -LD:=g++ - -DEBUG_PATH = build/debug -RELEASE_PATH = build/release -PROTO_SRC_DIR = build/if -MODULES = connection core serde test-util utils security exceptions -SRC_DIR = $(MODULES) -DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES)) -RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES)) -INCLUDE_DIR = . build/ $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux +#use "g++" to compile source files +CC := g++ +LD := g++ + +BUILD_PATH := build +DEBUG_PATH := $(BUILD_PATH)/debug +RELEASE_PATH := $(BUILD_PATH)/release +PROTO_SRC_DIR := if +PROTO_CXX_DIR := $(BUILD_PATH)/if +MODULES := connection core exceptions security serde test-util utils +SRC_DIR := $(MODULES) +DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES)) +RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES)) +INCLUDE_DIR := . $(BUILD_PATH) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux #flags to pass to the CPP compiler & linker -CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -LINKFLAG = -shared -L$(JAVA_HOME)/jre/lib/amd64/server -ljvm +CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP +CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP +LDFLAGS := -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -L $(JAVA_HOME)/jre/lib/amd64/server -ljvm +LINKFLAG := -shared #define list of source files and object files -ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc)) -EXCLUDE_SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc -SRC = $(filter-out $(EXCLUDE_SRC), $(ALLSRC)) -PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard if/*.proto)) -DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h)) -PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard if/*.proto)) -DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC)) +ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc)) +EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc +SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC)) +PROTOSRC := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.cc),$(wildcard if/*.proto)) +PROTOHDR := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.h),$(wildcard if/*.proto)) +DEBUG_OBJ := $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC)) DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC)) -RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC)) +RELEASE_OBJ := $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC)) RELEASE_OBJ += $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(PROTOSRC)) -INCLUDES = $(addprefix -I,$(INCLUDE_DIR)) - -LIB_DIR = /usr/local -LIB_LIBDIR = $(LIB_DIR)/lib -LIB_INCDIR = $(LIB_DIR)/include -LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so -ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a -LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so -ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a +INCLUDES := $(addprefix -I,$(INCLUDE_DIR)) + +LIB_DIR := /usr/local +LIB_LIBDIR := $(LIB_DIR)/lib +LIB_INCDIR := $(LIB_DIR)/include +LIB_RELEASE := $(RELEASE_PATH)/libHbaseClient.so +ARC_RELEASE := $(RELEASE_PATH)/libHbaseClient.a +LIB_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.so +ARC_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.a + +build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE) v
hbase git commit: HBASE-17782 Extend IdReadWriteLock to support using both weak and soft reference
Repository: hbase Updated Branches: refs/heads/master 14fb57cab -> aace02a23 HBASE-17782 Extend IdReadWriteLock to support using both weak and soft reference Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aace02a2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aace02a2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aace02a2 Branch: refs/heads/master Commit: aace02a230a61cc7e91eb240598435c36c9af403 Parents: 14fb57c Author: Yu Li Authored: Wed Mar 15 11:07:42 2017 +0800 Committer: Yu Li Committed: Wed Mar 15 11:07:42 2017 +0800 -- .../hbase/io/hfile/bucket/BucketCache.java | 5 +- .../hadoop/hbase/util/IdReadWriteLock.java | 58 .../hbase/wal/RegionGroupingProvider.java | 13 +++-- .../hadoop/hbase/util/TestIdReadWriteLock.java | 31 +-- 4 files changed, 86 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/aace02a2/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index cb23ca9..3e9c376 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.IdReadWriteLock; +import org.apache.hadoop.hbase.util.IdReadWriteLock.ReferenceType; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; @@ -185,9 +186,11 @@ public class BucketCache implements BlockCache, HeapSize { /** * A ReentrantReadWriteLock to lock on a particular block identified by offset. * The purpose of this is to avoid freeing the block which is being read. + * + * Key set of offsets in BucketCache is limited so soft reference is the best choice here. */ @VisibleForTesting - final IdReadWriteLock offsetLock = new IdReadWriteLock(); + final IdReadWriteLock offsetLock = new IdReadWriteLock(ReferenceType.SOFT); private final NavigableSet blocksByHFile = new ConcurrentSkipListSet<>(new Comparator() { http://git-wip-us.apache.org/repos/asf/hbase/blob/aace02a2/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java index deb2265..2a83029 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.util; +import java.lang.ref.Reference; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -44,16 +45,48 @@ import com.google.common.annotations.VisibleForTesting; public class IdReadWriteLock { // The number of lock we want to easily support. It's not a maximum. private static final int NB_CONCURRENT_LOCKS = 1000; - // The pool to get entry from, entries are mapped by soft reference and will be - // automatically garbage-collected when JVM memory pressure is high - private final ObjectPool lockPool = - new SoftObjectPool<>( - new ObjectPool.ObjectFactory() { -@Override -public ReentrantReadWriteLock createObject(Long id) { - return new ReentrantReadWriteLock(); -} - }, NB_CONCURRENT_LOCKS); + /** + * The pool to get entry from, entries are mapped by {@link Reference} and will be automatically + * garbage-collected by JVM + */ + private final ObjectPool lockPool; + private final ReferenceType refType; + + public IdReadWriteLock() { +this(ReferenceType.WEAK); + } + + /** + * Constructor of IdReadWriteLock + * @param referenceType type of the reference used in lock pool, {@link ReferenceType#WEAK} by + * default. Use {@link ReferenceType#SOFT} if the key set is limited and the locks will + * be reused with a high frequency + */ + public IdReadWriteLock(ReferenceType referenceType) { +this.refType = referenceType; +switch (referenceType) { +case SOFT: + lockPool = new SoftObjectPool<>(new ObjectPool.Ob