[kudu] branch master updated: [c++-client] KUDU-2671 more tests for unbounded ranges (CreateTable)
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new df51b540b [c++-client] KUDU-2671 more tests for unbounded ranges
(CreateTable)
df51b540b is described below
commit df51b540bb1f5fb40790389b77a90df589bf1363
Author: Alexey Serbin
AuthorDate: Fri Jul 22 15:59:37 2022 -0700
[c++-client] KUDU-2671 more tests for unbounded ranges (CreateTable)
This patch adds more tests for Kudu C++ client covering the paths
involving CreateTable RPC for tables with unbounded ranges having custom
hash schemas.
Change-Id: I584f613ddf3341156d02c4e7c9603b71584cfcf4
Reviewed-on: http://gerrit.cloudera.org:8080/18775
Reviewed-by: Mahesh Reddy
Tested-by: Kudu Jenkins
Reviewed-by: Yingchun Lai
Reviewed-by: Attila Bukor
---
src/kudu/client/flex_partitioning_client-test.cc | 91
1 file changed, 91 insertions(+)
diff --git a/src/kudu/client/flex_partitioning_client-test.cc
b/src/kudu/client/flex_partitioning_client-test.cc
index 1d3909940..3a89aea24 100644
--- a/src/kudu/client/flex_partitioning_client-test.cc
+++ b/src/kudu/client/flex_partitioning_client-test.cc
@@ -220,6 +220,13 @@ class FlexPartitioningTest : public KuduTest {
return CreateRangePartition(schema_, kKeyColumn, lower_bound, upper_bound);
}
+ RangePartition CreateRangePartitionNoLowerBound(int32_t upper_bound) {
+unique_ptr upper(schema_.NewRow());
+CHECK_OK(upper->SetInt32(kKeyColumn, upper_bound));
+return unique_ptr(
+new KuduRangePartition(schema_.NewRow(), upper.release()));
+ }
+
RangePartition CreateRangePartitionNoUpperBound(int32_t lower_bound) {
unique_ptr lower(schema_.NewRow());
CHECK_OK(lower->SetInt32(kKeyColumn, lower_bound));
@@ -1041,6 +1048,90 @@ TEST_F(FlexPartitioningCreateTableTest,
DISABLED_NoUpperBoundRangeCustomHashSche
NO_FATALS(CheckTableRowsNum(kTableName, 555));
}
+// Create a table unbounded ranges with custom hash schemas.
+TEST_F(FlexPartitioningCreateTableTest, UnboundedRangesWithCustomHashSchemas) {
+ constexpr const char* const kTableName =
+ "UnboundedRangesWithCustomHashSchemas";
+
+ unique_ptr table_creator(client_->NewTableCreator());
+ table_creator->table_name(kTableName)
+ .schema(&schema_)
+ .num_replicas(1)
+ .add_hash_partitions({ kKeyColumn }, 2)
+ .set_range_partition_columns({ kKeyColumn });
+
+ // Add a range partition with custom hash sub-partitioning rules:
+ // 3 buckets with hash based on the "key" column with hash seed 1.
+ {
+auto p = CreateRangePartitionNoLowerBound(0);
+ASSERT_OK(p->add_hash_partitions({ kKeyColumn }, 3, 1));
+table_creator->add_custom_range_partition(p.release());
+ }
+
+ // Add a range partition with custom hash sub-partitioning rules:
+ // 5 buckets with hash based on the "key" column with hash seed 2.
+ {
+auto p = CreateRangePartitionNoUpperBound(0);
+ASSERT_OK(p->add_hash_partitions({ kKeyColumn }, 5, 2));
+table_creator->add_custom_range_partition(p.release());
+ }
+
+ ASSERT_OK(table_creator->Create());
+ NO_FATALS(CheckTabletCount(kTableName, 8));
+
+ ASSERT_OK(InsertTestRows(kTableName, -250, -125));
+ NO_FATALS(CheckTableRowsNum(kTableName, 125));
+ ASSERT_OK(InsertTestRows(kTableName, 125, 250));
+ NO_FATALS(CheckTableRowsNum(kTableName, 250));
+ ASSERT_OK(InsertTestRows(kTableName, -125, 125));
+ NO_FATALS(CheckTableRowsNum(kTableName, 500));
+}
+
+// Similar to the UnboundedRangesWithCustomHashSchemas above, but with
+// additional range having table-wide table schema.
+TEST_F(FlexPartitioningCreateTableTest,
+ TwoUnboundedRangesWithCustomHashSchemasAndOneInBetween) {
+ constexpr const char* const kTableName =
+ "TwoUnboundedRangesWithCustomHashSchemasAndOneInBetween";
+
+ unique_ptr table_creator(client_->NewTableCreator());
+ table_creator->table_name(kTableName)
+ .schema(&schema_)
+ .num_replicas(1)
+ .add_hash_partitions({ kKeyColumn }, 3)
+ .set_range_partition_columns({ kKeyColumn });
+
+ // Add a range partition with the table-wide hash partitioning rules.
+ {
+unique_ptr lower(schema_.NewRow());
+ASSERT_OK(lower->SetInt32(kKeyColumn, -111));
+unique_ptr upper(schema_.NewRow());
+ASSERT_OK(upper->SetInt32(kKeyColumn, 111));
+table_creator->add_range_partition(lower.release(), upper.release());
+ }
+
+ // Add a range partition with custom hash sub-partitioning rules:
+ // 2 buckets with hash based on the "key" column with hash seed 1.
+ {
+auto p = CreateRangePartitionNoLowerBound(-111);
+ASSERT_OK(p->add_hash_partitions({ kKeyColumn }, 2, 1));
+table_creator->add_custom_range_partition(p.release());
+ }
+ {
+auto p = CreateRangePartitionNoUpperBound(111);
+ASSERT_OK(p->add_hash_partitions({ kKeyColumn
[kudu] branch master updated: [tool] Add partitions show with tablet id
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new a6e8aa2fd [tool] Add partitions show with tablet id
a6e8aa2fd is described below
commit a6e8aa2fd5063abb2cc5c8944c9d19fdb38b8aa0
Author: kedeng
AuthorDate: Tue Jun 14 15:43:05 2022 +0800
[tool] Add partitions show with tablet id
With this commit help, we can get the corresponding relationship
between partitions and tablet id. Which will help us to observe
the partitions more easily.
The command looks like:
`kudu table list --list_tablets --show_tablet_partition_info
[-negotiation_timeout_ms=] [-timeout_ms=]`
The output of the command looks like:
`
TestTableListPartition
T e72c529ad1a8481390683fd0fe3fb917 : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 674c732628b74fbab0a03a11f357b432 : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 1c9a5a6f62d1402aba04dc84f32b6b04 : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 44046ee1b43a43f4b59dad75f611ac0a : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 12ca618464074d8a9ae3f9581701bcf0 : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T edc1cbdae0934df9a48866fe65cfbec8 : HASH (key_hash0) PARTITION 0, HASH
(key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 91ebccd823634878a6e0a8a4e941aa4f : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 0057ad8aea0d4ad3a626b6417f14bcac : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 0, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T de3f9629a97148069aae9e598cb0d3fa : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T d7f9c9103f1943ef9b5131f03e9f8738 : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 1, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 63777094ede14232a5822a3ac50925fd : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 0 <= VALUES < 1
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
T 87f91f80595140fd9c9582a186a3f52b : HASH (key_hash0) PARTITION 1, HASH
(key_hash1, key_hash2) PARTITION 2, RANGE (key_range) PARTITION 2 <= VALUES < 3
L 8afe7243874943058e849fecebb84d98 127.28.231.1:11503
`
Change-Id: Ia9c3b62111c7f302ea934324d1636f79731f18f1
Reviewed-on: http://gerrit.cloudera.org:8080/18617
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin
---
src/kudu/client/client-internal.cc | 23 +-
src/kudu/client/client-internal.h| 12 ++-
src/kudu/common/partition.h | 2 -
src/kudu/integration-tests/cluster_itest_util.cc | 18
src/kudu/integration-tests/cluster_itest_util.h | 7 ++
src/kudu/master/catalog_manager.cc | 39 ++---
src/kudu/master/catalog_manager.h| 7 ++
src/kudu/master/master.proto | 10 +++
src/kudu/tools/kudu-admin-test.cc| 100 +++
src/kudu/tools/tool_action_table.cc | 50 +++-
10 files changed, 248 insertions(+), 20 deletions(-)
diff --git a/src/kudu/client/client-internal.cc
b/src/kudu/client/client-internal.cc
index 5c650599f..9095ae9c2 100644
--- a/src/kudu/client/client-internal.cc
+++ b/src/kudu/client/client-internal.cc
@@ -444,12 +444,15 @@ Status KuduClient::Data::WaitForAlterTableToFinish(
Status KuduClient::Data::ListTablesWithInfo(KuduClient* client,
vector* tables_info,
-const string& filter) {
+const st
[kudu] branch master updated: [java] KUDU-2671 fix bug in PartitionSchema::getHashSchemaForRange()
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 3b91db407 [java] KUDU-2671 fix bug in
PartitionSchema::getHashSchemaForRange()
3b91db407 is described below
commit 3b91db407d9973f154567e0b280f7fe39899c557
Author: Alexey Serbin
AuthorDate: Mon Jul 25 16:26:09 2022 -0700
[java] KUDU-2671 fix bug in PartitionSchema::getHashSchemaForRange()
This patch fixes a bug in PartitionSchema::getHashSchemaForRange()
that manifested itself when a right-side unbounded range with custom
hash schema was present in a table.
The patch also contains test scenarios that allowed to see the
manifestation of the bug before the fix; from now they allow to spot
regressions in the corresponding code, if any. These scenarios cover
both {Create,Alter}Table code paths when adding right-side unbounded
ranges with custom hash schemas.
Change-Id: Ib0ef5bbe4528ce5c1d4c9591a8152cf7ec4af0bb
Reviewed-on: http://gerrit.cloudera.org:8080/18782
Tested-by: Kudu Jenkins
Reviewed-by: Attila Bukor
---
.../org/apache/kudu/client/PartitionSchema.java| 11 ++-
.../org/apache/kudu/client/TestAlterTable.java | 110 +
.../java/org/apache/kudu/client/TestKuduTable.java | 89 +
3 files changed, 206 insertions(+), 4 deletions(-)
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
index e62ac8dbf..50851a4ae 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
@@ -205,7 +205,8 @@ public class PartitionSchema {
/**
* Find hash schema for the given encoded range key. Depending on the
* partition schema and the key, it might be either table-wide or a custom
- * hash schema for a particular range.
+ * hash schema for a particular range. Just as a convention, this method
+ * returns the table-wide hash schema for keys in non-covered ranges.
*
* @return hash bucket schema for the encoded range key
*/
@@ -221,10 +222,12 @@ public class PartitionSchema {
if (entry == null) {
return hashBucketSchemas;
}
-// Check if 'rangeKey' is in the range (null upper boundary means unbounded
-// range partition).
+// Check if 'rangeKey' is in the range.
+// NOTE: the right boundary is exclusive; an empty array for upper boundary
+// means that the range partition is unbounded.
final byte[] upper = entry.upper;
-if (upper == null || Bytes.memcmp(rangeKey, upper) < 0) {
+Preconditions.checkNotNull(upper);
+if (upper.length == 0 || Bytes.memcmp(rangeKey, upper) < 0) {
return entry.hashSchemas;
}
return hashBucketSchemas;
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
index 7e722e0de..f0270b446 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
@@ -599,6 +599,116 @@ public class TestAlterTable {
client.deleteTable(tableName);
}
+ /**
+ * Test altering a table, adding unbounded range partitions
+ * with custom hash schema.
+ */
+ @Test(timeout = 10)
+ public void testAlterAddUnboundedRangeWithCustomHashSchema() throws
Exception {
+ArrayList columns = new ArrayList<>(2);
+columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT32)
+.nullable(false)
+.key(true)
+.build());
+columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT32)
+.nullable(false)
+.build());
+final Schema schema = new Schema(columns);
+
+CreateTableOptions createOptions =
+new CreateTableOptions()
+.setRangePartitionColumns(ImmutableList.of("c0"))
+.addHashPartitions(ImmutableList.of("c0"), 2, 0)
+.setNumReplicas(1);
+// Add range partition [-100, 100) with the table-wide hash schema
+// (to be added upon creating the new table below).
+{
+ PartialRow lower = schema.newPartialRow();
+ lower.addInt("c0", -100);
+ PartialRow upper = schema.newPartialRow();
+ upper.addInt("c0", 100);
+ createOptions.addRangePartition(lower, upper);
+}
+// Add unbounded range partition [100, +inf) with custom hash schema.
+{
+ PartialRow lower = schema.newPartialRow();
+ lower.addInt("c0", 100);
+ PartialRow upper = schema.newPartialRow();
+ RangePartitionWithCustomHashSchema range =
+ new RangePartitionWithCustomHashSchema(
+ lower,
+
