This is an automated email from the ASF dual-hosted git repository. alexey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push: new 76d540c13 [tool] Add gflag to control the display of hash info when show partition info. 76d540c13 is described below commit 76d540c137c991127865d5d2455d113677a9afe9 Author: kedeng <kdeng...@gmail.com> AuthorDate: Fri Jul 29 11:52:54 2022 +0800 [tool] Add gflag to control the display of hash info when show partition info. The command with 'show_hash_partition_info' looks like: `kudu table list --list_tablets --show_tablet_partition_info --show_hash_partition_info <table_name> <master_addresses> [-negotiation_timeout_ms=<ms>] [-timeout_ms=<ms>]` The output of the command with 'show_hash_partition_info' looks like: ` TestTableListPartition T f7537632388b46a394b818979a17920c : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 4f1a71d9907b49a1aff479fbafc65a42 : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T b11db1b8fc304cc19a0211df45818efc : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T db401470d3374d9bba75f56677c1d1c2 : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T ead7e1bbc3cd465cb755118e769b8c72 : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 19ffc549b26f49928ecb552224860bf2 : HASH (key_hash0) PARTITION 0, HASH (key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 38c95bd395404dbda094826f5054aced : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 8c257d4c5ebc4bffb5c27beeef1e9355 : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 6c6d1f1e4df8456f9853beaab6bd7bee : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 05f27b37443c4b7e843e1d7bba2fb5ee : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 48ece2705bc14819b1a38aa489f39e50 : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 768cea86cf8543258313e242afa8f057 : HASH (key_hash0) PARTITION 1, HASH (key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 ` The command without 'show_hash_partition_info' looks like: `kudu table list --list_tablets --show_tablet_partition_info <table_name> <master_addresses> [-negotiation_timeout_ms=<ms>] [-timeout_ms=<ms>]` The output of the command without 'show_hash_partition_info' looks like: ` TestTableListPartition T f7537632388b46a394b818979a17920c : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 4f1a71d9907b49a1aff479fbafc65a42 : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T b11db1b8fc304cc19a0211df45818efc : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T db401470d3374d9bba75f56677c1d1c2 : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T ead7e1bbc3cd465cb755118e769b8c72 : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 19ffc549b26f49928ecb552224860bf2 : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 38c95bd395404dbda094826f5054aced : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 8c257d4c5ebc4bffb5c27beeef1e9355 : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 6c6d1f1e4df8456f9853beaab6bd7bee : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 05f27b37443c4b7e843e1d7bba2fb5ee : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 48ece2705bc14819b1a38aa489f39e50 : RANGE (key_range) PARTITION 0 <= VALUES < 1 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 T 768cea86cf8543258313e242afa8f057 : RANGE (key_range) PARTITION 2 <= VALUES < 3 L 883ba129547f4e9a90134fac739334f1 127.18.13.1:34907 ` Change-Id: I8bba23740a8544ea40360b70e394c31d500f81c1 Reviewed-on: http://gerrit.cloudera.org:8080/18797 Tested-by: Kudu Jenkins Reviewed-by: Yingchun Lai <acelyc1112...@gmail.com> Reviewed-by: Alexey Serbin <ale...@apache.org> --- src/kudu/common/partition-test.cc | 36 +++++++++++++++++++++++++++++++++++- src/kudu/common/partition.cc | 15 +++++++++------ src/kudu/common/partition.h | 9 ++++++++- src/kudu/tools/kudu-admin-test.cc | 17 +++++++++++++---- src/kudu/tools/tool_action_table.cc | 20 +++++++++++++++++++- 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/kudu/common/partition-test.cc b/src/kudu/common/partition-test.cc index b992a3957..b0db9208c 100644 --- a/src/kudu/common/partition-test.cc +++ b/src/kudu/common/partition-test.cc @@ -577,7 +577,6 @@ TEST_F(PartitionTest, TestCreatePartitions) { R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", partition_schema.PartitionDebugString(partitions[0], schema)); - EXPECT_EQ(0, partitions[1].hash_buckets()[0]); EXPECT_EQ(0, partitions[1].hash_buckets()[1]); EXPECT_EQ(string("a1\0\0b1\0\0c1", 10), partitions[1].begin().range_key()); @@ -589,6 +588,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 0, HASH (b) PARTITION 0, " R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", partition_schema.PartitionDebugString(partitions[1], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", + partition_schema.PartitionDebugString(partitions[1], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(0, partitions[2].hash_buckets()[0]); EXPECT_EQ(0, partitions[2].hash_buckets()[1]); @@ -599,6 +601,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 0, HASH (b) PARTITION 0, " R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", partition_schema.PartitionDebugString(partitions[2], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", + partition_schema.PartitionDebugString(partitions[2], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(0, partitions[3].hash_buckets()[0]); EXPECT_EQ(1, partitions[3].hash_buckets()[1]); @@ -609,6 +614,10 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 0, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", partition_schema.PartitionDebugString(partitions[3], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", + partition_schema.PartitionDebugString(partitions[3], schema, + PartitionSchema::HashPartitionInfo::HIDE)); + EXPECT_EQ(0, partitions[4].hash_buckets()[0]); EXPECT_EQ(1, partitions[4].hash_buckets()[1]); @@ -620,6 +629,10 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 0, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", partition_schema.PartitionDebugString(partitions[4], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", + partition_schema.PartitionDebugString(partitions[4], schema, + PartitionSchema::HashPartitionInfo::HIDE)); + EXPECT_EQ(0, partitions[5].hash_buckets()[0]); EXPECT_EQ(1, partitions[5].hash_buckets()[1]); @@ -630,6 +643,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 0, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", partition_schema.PartitionDebugString(partitions[5], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", + partition_schema.PartitionDebugString(partitions[5], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[6].hash_buckets()[0]); EXPECT_EQ(0, partitions[6].hash_buckets()[1]); @@ -640,6 +656,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 0, " R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", partition_schema.PartitionDebugString(partitions[6], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", + partition_schema.PartitionDebugString(partitions[6], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[7].hash_buckets()[0]); EXPECT_EQ(0, partitions[7].hash_buckets()[1]); @@ -651,6 +670,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 0, " R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", partition_schema.PartitionDebugString(partitions[7], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", + partition_schema.PartitionDebugString(partitions[7], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[8].hash_buckets()[0]); EXPECT_EQ(0, partitions[8].hash_buckets()[1]); @@ -661,6 +683,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 0, " R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", partition_schema.PartitionDebugString(partitions[8], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", + partition_schema.PartitionDebugString(partitions[8], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[9].hash_buckets()[0]); EXPECT_EQ(1, partitions[9].hash_buckets()[1]); @@ -671,6 +696,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", partition_schema.PartitionDebugString(partitions[9], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION VALUES < ("a1", "b1", "c1"))", + partition_schema.PartitionDebugString(partitions[9], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[10].hash_buckets()[0]); EXPECT_EQ(1, partitions[10].hash_buckets()[1]); @@ -682,6 +710,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", partition_schema.PartitionDebugString(partitions[10], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a1", "b1", "c1") <= VALUES < ("a2", "b2", ""))", + partition_schema.PartitionDebugString(partitions[10], schema, + PartitionSchema::HashPartitionInfo::HIDE)); EXPECT_EQ(1, partitions[11].hash_buckets()[0]); EXPECT_EQ(1, partitions[11].hash_buckets()[1]); @@ -692,6 +723,9 @@ TEST_F(PartitionTest, TestCreatePartitions) { EXPECT_EQ("HASH (a) PARTITION 1, HASH (b) PARTITION 1, " R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", partition_schema.PartitionDebugString(partitions[11], schema)); + EXPECT_EQ(R"(RANGE (a, b, c) PARTITION ("a2", "b2", "") <= VALUES)", + partition_schema.PartitionDebugString(partitions[11], schema, + PartitionSchema::HashPartitionInfo::HIDE)); } TEST_F(PartitionTest, TestIncrementRangePartitionBounds) { diff --git a/src/kudu/common/partition.cc b/src/kudu/common/partition.cc index 3d428999e..9aef2e4dc 100644 --- a/src/kudu/common/partition.cc +++ b/src/kudu/common/partition.cc @@ -901,7 +901,8 @@ string ColumnIdsToColumnNames(const Schema& schema, } // namespace string PartitionSchema::PartitionDebugString(const Partition& partition, - const Schema& schema) const { + const Schema& schema, + HashPartitionInfo hp) const { // Partitions are considered metadata, so don't redact them. ScopedDisableRedaction no_redaction; @@ -912,11 +913,13 @@ string PartitionSchema::PartitionDebugString(const Partition& partition, } vector<string> components; - for (size_t i = 0; i < hash_schema.size(); ++i) { - string s = Substitute("HASH ($0) PARTITION $1", - ColumnIdsToColumnNames(schema, hash_schema[i].column_ids), - partition.hash_buckets_[i]); - components.emplace_back(std::move(s)); + if (hp == HashPartitionInfo::SHOW) { + for (size_t i = 0; i < hash_schema.size(); ++i) { + string s = Substitute("HASH ($0) PARTITION $1", + ColumnIdsToColumnNames(schema, hash_schema[i].column_ids), + partition.hash_buckets_[i]); + components.emplace_back(std::move(s)); + } } if (!range_schema_.column_ids.empty()) { diff --git a/src/kudu/common/partition.h b/src/kudu/common/partition.h index 9fb886196..575c73218 100644 --- a/src/kudu/common/partition.h +++ b/src/kudu/common/partition.h @@ -382,11 +382,18 @@ class PartitionSchema { bool PartitionMayContainRow(const Partition& partition, const KuduPartialRow& row) const; + // Used to determine whether the partition keys information shows up. + enum HashPartitionInfo { + HIDE = 0, + SHOW = 1, + }; + // Returns a text description of the partition suitable for debug printing. // // Partitions are considered metadata, so no redaction will happen on the hash // and range bound values. - std::string PartitionDebugString(const Partition& partition, const Schema& schema) const; + std::string PartitionDebugString(const Partition& partition, const Schema& schema, + HashPartitionInfo hp = HashPartitionInfo::SHOW) const; // Returns a text description of a partition key suitable for debug printing. std::string PartitionKeyDebugString(const PartitionKey& key, diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc index 8602b74f4..4d9706b4c 100644 --- a/src/kudu/tools/kudu-admin-test.cc +++ b/src/kudu/tools/kudu-admin-test.cc @@ -2144,7 +2144,14 @@ TEST_F(AdminCliTest, TestDescribeTableCustomHashSchema) { " PARTITION 100 <= VALUES < 200"); } -TEST_F(AdminCliTest, TestListTabletWithPartition) { +class ListTableCliParamTest : public AdminCliTest, + public ::testing::WithParamInterface<bool> { +}; + +// Basic test that the kudu tool works in the list tablets case. +TEST_P(ListTableCliParamTest, TestListTabletWithPartition) { + auto show_hp = GetParam() ? PartitionSchema::HashPartitionInfo::SHOW : + PartitionSchema::HashPartitionInfo::HIDE; FLAGS_num_tablet_servers = 1; FLAGS_num_replicas = 1; @@ -2212,7 +2219,7 @@ TEST_F(AdminCliTest, TestListTabletWithPartition) { "table", "list", "--list_tablets", - "--show_tablet_partition_info", + GetParam() ? "--show_tablet_partition_info" : "", "--tables", kTableId, cluster_->master()->bound_rpc_addr().ToString(), @@ -2229,12 +2236,14 @@ TEST_F(AdminCliTest, TestListTabletWithPartition) { master::ListTablesResponsePB tables_info; ASSERT_OK(ListTablesWithInfo(cluster_->master_proxy(), kTableId, - MonoDelta::FromSeconds(30), &tables_info)); + MonoDelta::FromSeconds(30), &tables_info)); for (const auto& table : tables_info.tables()) { for (const auto& pt : table.tablet_with_partition()) { Partition partition; Partition::FromPB(pt.partition(), &partition); - string partition_str = partition_schema.PartitionDebugString(partition, schema_internal); + string partition_str = partition_schema.PartitionDebugString(partition, + schema_internal, + show_hp); string tablet_with_partition = pt.tablet_id() + " : " + partition_str; ASSERT_STR_CONTAINS(stdout, tablet_with_partition); } diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc index ad4dd3053..a03c23574 100644 --- a/src/kudu/tools/tool_action_table.cc +++ b/src/kudu/tools/tool_action_table.cc @@ -123,6 +123,20 @@ bool ValidateShowTabletPartitionInfo() { GROUP_FLAG_VALIDATOR(show_tablet_partition_info, ValidateShowTabletPartitionInfo); +DEFINE_bool(show_hash_partition_info, false, + "Include hash partition keys information corresponding to tablet in the output."); + +bool ValidateShowHashPartitionInfo() { + if (!FLAGS_show_tablet_partition_info && FLAGS_show_tablet_partition_info) { + LOG(ERROR) << Substitute("--show_hash_partition_info is meaningless " + "when --show_tablet_partition_info=false"); + return false; + } + return true; +} + +GROUP_FLAG_VALIDATOR(show_hash_partition_info, ValidateShowHashPartitionInfo); + DEFINE_bool(modify_external_catalogs, true, "Whether to modify external catalogs, such as the Hive Metastore, " "when renaming or dropping a table."); @@ -193,7 +207,10 @@ class TableLister { const auto& schema_internal = KuduSchema::ToSchema(table->schema()); const auto& partition_schema = table->partition_schema(); - pinfo = partition_schema.PartitionDebugString(pt.partition, schema_internal); + auto show_hp = FLAGS_show_hash_partition_info ? PartitionSchema::HashPartitionInfo::SHOW : + PartitionSchema::HashPartitionInfo::HIDE; + pinfo = partition_schema.PartitionDebugString(pt.partition, schema_internal, + show_hp); break; } return pinfo; @@ -1604,6 +1621,7 @@ unique_ptr<Mode> BuildTableMode() { .AddOptionalParameter("tables") .AddOptionalParameter("list_tablets") .AddOptionalParameter("show_tablet_partition_info") + .AddOptionalParameter("show_hash_partition_info") .AddOptionalParameter("show_table_info") .Build();