HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync. Summary: On every start of docker make sure that protos are up to date.
Test Plan: Added the script and it updated the protos, but didn't delete BUCK. Differential Revision: https://reviews.facebook.net/D56763 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f3442379 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f3442379 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f3442379 Branch: refs/heads/HBASE-14850 Commit: f3442379616c5874504695d5702e2d701ab3c512 Parents: 17185fe Author: Elliott Clark <ecl...@apache.org> Authored: Wed Apr 13 15:59:13 2016 -0700 Committer: Elliott Clark <elli...@fb.com> Committed: Wed May 18 15:48:52 2016 -0700 ---------------------------------------------------------------------- hbase-native-client/bin/copy-protobuf.sh | 8 ++ hbase-native-client/bin/start-docker.sh | 22 +++- hbase-native-client/if/Admin.proto | 1 + hbase-native-client/if/Client.proto | 8 +- hbase-native-client/if/ClusterStatus.proto | 3 + hbase-native-client/if/Filter.proto | 1 + hbase-native-client/if/Master.proto | 47 +++++++- hbase-native-client/if/MasterProcedure.proto | 40 +++++++ hbase-native-client/if/RSGroup.proto | 34 ++++++ hbase-native-client/if/RSGroupAdmin.proto | 136 ++++++++++++++++++++++ hbase-native-client/if/ZooKeeper.proto | 13 +++ 11 files changed, 304 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/bin/copy-protobuf.sh ---------------------------------------------------------------------- diff --git a/hbase-native-client/bin/copy-protobuf.sh b/hbase-native-client/bin/copy-protobuf.sh new file mode 100755 index 0000000..c9e70f2 --- /dev/null +++ b/hbase-native-client/bin/copy-protobuf.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +BIN_DIR=$(dirname "$0") +PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/" +PB_DEST_DIR="${BIN_DIR}/../if/" +rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR} http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/bin/start-docker.sh ---------------------------------------------------------------------- diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh index 725ed6a..9cbd8b7 100755 --- a/hbase-native-client/bin/start-docker.sh +++ b/hbase-native-client/bin/start-docker.sh @@ -23,22 +23,38 @@ set -x eval "$(docker-machine env docker-vm)" eval "$(docker-machine env dinghy)" -# Build the image -docker build -t hbase_native . +BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null) +BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null) +${BIN_DIR}/copy-protobuf.sh +# Go into the base dir. This just makes things cleaner. +pushd ${BASE_DIR} + +# Make sure that there is a thrid-party dir. mkdir third-party || true + +# Get gtest +# TODO(eclark): Remove this ( see HBASE-15427 ) if [[ ! -d third-party/googletest ]]; then git clone https://github.com/google/googletest.git third-party/googletest fi +# We don't want to have to re-download all the jars in docker if we can help it if [[ ! -d ~/.m2 ]]; then echo "~/.m2 directory doesn't exist. Check Apache Maven is installed." exit 1 fi; +# Build the image +# +# This shouldn't be needed after the development environment is a little more stable. +docker build -t hbase_native . + +# After the image is built run the thing docker run -p 16010:16010/tcp \ -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \ - -v ${PWD}/..:/usr/src/hbase \ + -v ${BASE_DIR}/..:/usr/src/hbase \ -v ~/.m2:/root/.m2 \ -it hbase_native /bin/bash +popd http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Admin.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/Admin.proto b/hbase-native-client/if/Admin.proto index e905340..a1905a4 100644 --- a/hbase-native-client/if/Admin.proto +++ b/hbase-native-client/if/Admin.proto @@ -25,6 +25,7 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; +import "Client.proto"; import "HBase.proto"; import "WAL.proto"; http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Client.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/Client.proto b/hbase-native-client/if/Client.proto index ca9abdc..8a4d459 100644 --- a/hbase-native-client/if/Client.proto +++ b/hbase-native-client/if/Client.proto @@ -402,6 +402,11 @@ message RegionLoadStats { optional int32 compactionPressure = 3 [default = 0]; } +message MultiRegionLoadStats{ + repeated RegionSpecifier region = 1; + repeated RegionLoadStats stat = 2; +} + /** * Either a Result or an Exception NameBytesPair (keyed by * exception name whose value is the exception stringified) @@ -416,7 +421,7 @@ message ResultOrException { // result if this was a coprocessor service call optional CoprocessorServiceResult service_result = 4; // current load on the region - optional RegionLoadStats loadStats = 5; + optional RegionLoadStats loadStats = 5 [deprecated=true]; } /** @@ -445,6 +450,7 @@ message MultiResponse { repeated RegionActionResult regionActionResult = 1; // used for mutate to indicate processed only optional bool processed = 2; + optional MultiRegionLoadStats regionStatistics = 3; } http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/ClusterStatus.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/ClusterStatus.proto b/hbase-native-client/if/ClusterStatus.proto index 228be7e..54bc0c3 100644 --- a/hbase-native-client/if/ClusterStatus.proto +++ b/hbase-native-client/if/ClusterStatus.proto @@ -136,6 +136,9 @@ message RegionLoad { /** the most recent sequence Id of store from cache flush */ repeated StoreSequenceId store_complete_sequence_id = 18; + + /** the current total filtered read requests made to region */ + optional uint64 filtered_read_requests_count = 19; } /* Server-level protobufs */ http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Filter.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/Filter.proto b/hbase-native-client/if/Filter.proto index 67d5717..1fa6697 100644 --- a/hbase-native-client/if/Filter.proto +++ b/hbase-native-client/if/Filter.proto @@ -146,6 +146,7 @@ message SkipFilter { message TimestampsFilter { repeated int64 timestamps = 1 [packed=true]; + optional bool can_hint = 2; } message ValueFilter { http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Master.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto index 4d3a2e1..1f7a3b7 100644 --- a/hbase-native-client/if/Master.proto +++ b/hbase-native-client/if/Master.proto @@ -186,6 +186,7 @@ message CreateNamespaceRequest { } message CreateNamespaceResponse { + optional uint64 proc_id = 1; } message DeleteNamespaceRequest { @@ -195,6 +196,7 @@ message DeleteNamespaceRequest { } message DeleteNamespaceResponse { + optional uint64 proc_id = 1; } message ModifyNamespaceRequest { @@ -204,6 +206,7 @@ message ModifyNamespaceRequest { } message ModifyNamespaceResponse { + optional uint64 proc_id = 1; } message GetNamespaceDescriptorRequest { @@ -276,6 +279,29 @@ message IsBalancerEnabledResponse { required bool enabled = 1; } +enum MasterSwitchType { + SPLIT = 0; + MERGE = 1; +} + +message SetSplitOrMergeEnabledRequest { + required bool enabled = 1; + optional bool synchronous = 2; + repeated MasterSwitchType switch_types = 3; +} + +message SetSplitOrMergeEnabledResponse { + repeated bool prev_value = 1; +} + +message IsSplitOrMergeEnabledRequest { + required MasterSwitchType switch_type = 1; +} + +message IsSplitOrMergeEnabledResponse { + required bool enabled = 1; +} + message NormalizeRequest { } @@ -344,9 +370,12 @@ message DeleteSnapshotResponse { message RestoreSnapshotRequest { required SnapshotDescription snapshot = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; } message RestoreSnapshotResponse { + required uint64 proc_id = 1; } /* if you don't send the snapshot, then you will get it back @@ -630,6 +659,19 @@ service MasterService { returns(IsBalancerEnabledResponse); /** + * Turn the split or merge switch on or off. + * If synchronous is true, it waits until current operation call, if outstanding, to return. + */ + rpc SetSplitOrMergeEnabled(SetSplitOrMergeEnabledRequest) + returns(SetSplitOrMergeEnabledResponse); + + /** + * Query whether the split or merge switch is on/off. + */ + rpc IsSplitOrMergeEnabled(IsSplitOrMergeEnabledRequest) + returns(IsSplitOrMergeEnabledResponse); + + /** * Run region normalizer. Can NOT run for various reasons. Check logs. */ rpc Normalize(NormalizeRequest) @@ -696,11 +738,6 @@ service MasterService { rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse); /** - * Determine if the snapshot restore is done yet. - */ - rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse); - - /** * Execute a distributed procedure. */ rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse); http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/MasterProcedure.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/MasterProcedure.proto b/hbase-native-client/if/MasterProcedure.proto index 2d2aff4..87aae6a 100644 --- a/hbase-native-client/if/MasterProcedure.proto +++ b/hbase-native-client/if/MasterProcedure.proto @@ -222,6 +222,46 @@ message DisableTableStateData { required bool skip_table_state_check = 3; } +message RestoreParentToChildRegionsPair { + required string parent_region_name = 1; + required string child1_region_name = 2; + required string child2_region_name = 3; +} + +enum CloneSnapshotState { + CLONE_SNAPSHOT_PRE_OPERATION = 1; + CLONE_SNAPSHOT_WRITE_FS_LAYOUT = 2; + CLONE_SNAPSHOT_ADD_TO_META = 3; + CLONE_SNAPSHOT_ASSIGN_REGIONS = 4; + CLONE_SNAPSHOT_UPDATE_DESC_CACHE = 5; + CLONE_SNAPSHOT_POST_OPERATION = 6; +} + +message CloneSnapshotStateData { + required UserInformation user_info = 1; + required SnapshotDescription snapshot = 2; + required TableSchema table_schema = 3; + repeated RegionInfo region_info = 4; + repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5; +} + +enum RestoreSnapshotState { + RESTORE_SNAPSHOT_PRE_OPERATION = 1; + RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR = 2; + RESTORE_SNAPSHOT_WRITE_FS_LAYOUT = 3; + RESTORE_SNAPSHOT_UPDATE_META = 4; +} + +message RestoreSnapshotStateData { + required UserInformation user_info = 1; + required SnapshotDescription snapshot = 2; + required TableSchema modified_table_schema = 3; + repeated RegionInfo region_info_for_restore = 4; + repeated RegionInfo region_info_for_remove = 5; + repeated RegionInfo region_info_for_add = 6; + repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 7; +} + message ServerCrashStateData { required ServerName server_name = 1; optional bool distributed_log_replay = 2; http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/RSGroup.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/RSGroup.proto b/hbase-native-client/if/RSGroup.proto new file mode 100644 index 0000000..7358941 --- /dev/null +++ b/hbase-native-client/if/RSGroup.proto @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RSGroupProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message RSGroupInfo { + required string name = 1; + repeated ServerName servers = 4; + repeated TableName tables = 3; +} + http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/RSGroupAdmin.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/RSGroupAdmin.proto b/hbase-native-client/if/RSGroupAdmin.proto new file mode 100644 index 0000000..fda9b09 --- /dev/null +++ b/hbase-native-client/if/RSGroupAdmin.proto @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RSGroupAdminProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "RSGroup.proto"; + +/** Group level protobufs */ + +message ListTablesOfRSGroupRequest { + required string r_s_group_name = 1; +} + +message ListTablesOfRSGroupResponse { + repeated TableName table_name = 1; +} + +message GetRSGroupInfoRequest { + required string r_s_group_name = 1; +} + +message GetRSGroupInfoResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfTableRequest { + required TableName table_name = 1; +} + +message GetRSGroupInfoOfTableResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message MoveServersRequest { + required string target_group = 1; + repeated ServerName servers = 3; +} + +message MoveServersResponse { +} + +message MoveTablesRequest { + required string target_group = 1; + repeated TableName table_name = 2; +} + +message MoveTablesResponse { +} + +message AddRSGroupRequest { + required string r_s_group_name = 1; +} + +message AddRSGroupResponse { +} + +message RemoveRSGroupRequest { + required string r_s_group_name = 1; +} + +message RemoveRSGroupResponse { +} + +message BalanceRSGroupRequest { + required string r_s_group_name = 1; +} + +message BalanceRSGroupResponse { + required bool balanceRan = 1; +} + +message ListRSGroupInfosRequest { +} + +message ListRSGroupInfosResponse { + repeated RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfServerRequest { + required ServerName server = 2; +} + +message GetRSGroupInfoOfServerResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +service RSGroupAdminService { + rpc GetRSGroupInfo(GetRSGroupInfoRequest) + returns (GetRSGroupInfoResponse); + + rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest) + returns (GetRSGroupInfoOfTableResponse); + + rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest) + returns (GetRSGroupInfoOfServerResponse); + + rpc MoveServers(MoveServersRequest) + returns (MoveServersResponse); + + rpc MoveTables(MoveTablesRequest) + returns (MoveTablesResponse); + + rpc AddRSGroup(AddRSGroupRequest) + returns (AddRSGroupResponse); + + rpc RemoveRSGroup(RemoveRSGroupRequest) + returns (RemoveRSGroupResponse); + + rpc BalanceRSGroup(BalanceRSGroupRequest) + returns (BalanceRSGroupResponse); + + rpc ListRSGroupInfos(ListRSGroupInfosRequest) + returns (ListRSGroupInfosResponse); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/ZooKeeper.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto index 54652af..186d183 100644 --- a/hbase-native-client/if/ZooKeeper.proto +++ b/hbase-native-client/if/ZooKeeper.proto @@ -105,6 +105,11 @@ message DeprecatedTableState { required State state = 1 [default = ENABLED]; } +message TableCF { + optional TableName table_name = 1; + repeated bytes families = 2; +} + /** * Used by replication. Holds a replication peer key. */ @@ -115,6 +120,7 @@ message ReplicationPeer { optional string replicationEndpointImpl = 2; repeated BytesBytesPair data = 3; repeated NameStringPair configuration = 4; + repeated TableCF table_cfs = 5; } /** @@ -153,3 +159,10 @@ message TableLock { optional string purpose = 5; optional int64 create_time = 6; } + +/** + * State of the switch. + */ +message SwitchState { + optional bool enabled = 1; +} \ No newline at end of file
