[kudu] branch master updated: KUDU-2671 introduce RANGE_SPECIFIC_HASH_SCHEMA feature flag
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 76f475aba KUDU-2671 introduce RANGE_SPECIFIC_HASH_SCHEMA feature flag
76f475aba is described below
commit 76f475abad0194464fd2e46383be7467f50aedd3
Author: Alexey Serbin
AuthorDate: Thu Jun 16 19:15:43 2022 -0700
KUDU-2671 introduce RANGE_SPECIFIC_HASH_SCHEMA feature flag
This patch introduces a new RANGE_SPECIFIC_HASH_SCHEMA flag for master
to signal that a Kudu cluster is able to work with tables having
range-specific hash schemas (a.k.a. custom hash schemas per range).
In addition, now C++ client requires the new flag to be present at
the server side when creating a table having at least one range
partition with custom hash schema or when adding a new range partition
with custom hash schema.
The rationale for introducing the flag is the following: if there were
no RANGE_SPECIFIC_HASH_SCHEMA flag and a newer client were not requiring
the server to have such a flag, the client would not get an error while
trying to perform the following operations against tablet servers
of prior versions:
* Creating a table having a range partition with custom hash schema
* Adding a range partition with custom hash schema to existing table
That's because the information on custom hash schemas is provided via
newly added flags in corresponding protobuf structures, and the old
server would simply ignore the fields, assuming all the ranges to be
created have the table-wide hash schema.
A follow-up patch will add similar functionality for Kudu Java client.
Change-Id: I256d32003e869939e7aa581b21bbe1e77c1e3aba
Reviewed-on: http://gerrit.cloudera.org:8080/18633
Reviewed-by: Mahesh Reddy
Reviewed-by: Abhishek Chennaka
Tested-by: Alexey Serbin
Reviewed-by: Attila Bukor
---
src/kudu/client/client-internal.cc | 12 +-
src/kudu/client/client-internal.h| 6 +-
src/kudu/client/client.cc| 21 +++-
src/kudu/client/flex_partitioning_client-test.cc | 138 +++
src/kudu/client/table_alterer-internal.h | 3 +
src/kudu/master/master.proto | 2 +
src/kudu/master/master_service.cc| 3 +
7 files changed, 175 insertions(+), 10 deletions(-)
diff --git a/src/kudu/client/client-internal.cc
b/src/kudu/client/client-internal.cc
index 99934e28f..5c650599f 100644
--- a/src/kudu/client/client-internal.cc
+++ b/src/kudu/client/client-internal.cc
@@ -316,11 +316,15 @@ Status KuduClient::Data::CreateTable(KuduClient* client,
const CreateTableRequestPB& req,
CreateTableResponsePB* resp,
const MonoTime& deadline,
- bool has_range_partition_bounds) {
+ bool has_range_partition_bounds,
+ bool has_range_specific_hash_schema) {
vector features;
if (has_range_partition_bounds) {
features.push_back(MasterFeatures::RANGE_PARTITION_BOUNDS);
}
+ if (has_range_specific_hash_schema) {
+features.push_back(MasterFeatures::RANGE_SPECIFIC_HASH_SCHEMA);
+ }
Synchronizer sync;
AsyncLeaderMasterRpc rpc(
deadline, client, BackoffType::EXPONENTIAL, req, resp,
@@ -387,11 +391,15 @@ Status KuduClient::Data::AlterTable(KuduClient* client,
const AlterTableRequestPB& req,
AlterTableResponsePB* resp,
const MonoTime& deadline,
-bool has_add_drop_partition) {
+bool has_add_drop_partition,
+bool adding_range_with_custom_hash_schema)
{
vector required_feature_flags;
if (has_add_drop_partition) {
required_feature_flags.push_back(MasterFeatures::ADD_DROP_RANGE_PARTITIONS);
}
+ if (adding_range_with_custom_hash_schema) {
+
required_feature_flags.push_back(MasterFeatures::RANGE_SPECIFIC_HASH_SCHEMA);
+ }
Synchronizer sync;
AsyncLeaderMasterRpc rpc(
deadline, client, BackoffType::EXPONENTIAL, req, resp,
diff --git a/src/kudu/client/client-internal.h
b/src/kudu/client/client-internal.h
index deab47d42..61661f73b 100644
--- a/src/kudu/client/client-internal.h
+++ b/src/kudu/client/client-internal.h
@@ -105,7 +105,8 @@ class KuduClient::Data {
const master::CreateTableRequestPB& req,
master::CreateTableResponsePB* resp,
const MonoTime& deadline,
-bool has_range_partition_boun
[kudu] branch master updated: [java] KUDU-2671 support creating table with custom hash schema per range
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 8b31a0ed4 [java] KUDU-2671 support creating table with custom hash
schema per range
8b31a0ed4 is described below
commit 8b31a0ed48a8ee830c68dc74143c924f4b1f2116
Author: Alexey Serbin
AuthorDate: Thu Dec 2 21:07:44 2021 -0800
[java] KUDU-2671 support creating table with custom hash schema per range
This patch introduces changes in the Kudu Java client API to make it
possible to create a Kudu table with custom hash bucket schemas per
range partition. Corresponding test coverage is present as well.
This is a patch to complement 586b79132 at the Kudu Java client side
(586b79132 introduced corresponding changes at the Kudu C++ client).
The appropriate Spark bindings haven't been updated yet -- that
to be done in a separate changelist.
Change-Id: I5ccf77ea2c39808520e76351d62571d449d10894
Reviewed-on: http://gerrit.cloudera.org:8080/18562
Tested-by: Alexey Serbin
Reviewed-by: Zoltan Chovan
Reviewed-by: Attila Bukor
---
.../org/apache/kudu/client/AsyncKuduClient.java| 4 +-
.../org/apache/kudu/client/CreateTableOptions.java | 99 ++-
.../java/org/apache/kudu/client/KeyEncoder.java| 21 +-
.../java/org/apache/kudu/client/Operation.java | 24 +-
.../org/apache/kudu/client/PartitionPruner.java| 312 +--
.../org/apache/kudu/client/PartitionSchema.java| 163 +++-
.../org/apache/kudu/client/ProtobufHelper.java | 53 +-
.../org/apache/kudu/client/RangePartition.java | 66 ++
.../client/RangePartitionWithCustomHashSchema.java | 87 ++
.../org/apache/kudu/client/TestKeyEncoding.java| 59 +-
.../org/apache/kudu/client/TestKuduClient.java | 103 +++
.../java/org/apache/kudu/client/TestKuduTable.java | 902 +
.../java/org/apache/kudu/client/TestOperation.java | 26 +-
.../apache/kudu/client/TestPartitionPruner.java| 219 +
14 files changed, 1969 insertions(+), 169 deletions(-)
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 3a0780c14..1b1a690ca 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -633,8 +633,8 @@ public class AsyncKuduClient implements AutoCloseable {
if (builder == null) {
throw new IllegalArgumentException("CreateTableOptions may not be null");
}
-if (!builder.getBuilder().getPartitionSchema().hasRangeSchema() &&
-builder.getBuilder().getPartitionSchema().getHashSchemaCount() == 0) {
+final Common.PartitionSchemaPB ps =
builder.getBuilder().getPartitionSchema();
+if (!ps.hasRangeSchema() && ps.getHashSchemaCount() == 0) {
throw new IllegalArgumentException("Table partitioning must be specified
using " +
"setRangePartitionColumns or
addHashPartitions");
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
index 6ac83dd62..e1d35360c 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -38,8 +38,11 @@ public class CreateTableOptions {
private final List splitRows = Lists.newArrayList();
private final List rangePartitions = Lists.newArrayList();
+ private final List customRangePartitions
=
+ Lists.newArrayList(); // range partitions with custom hash schemas
private Master.CreateTableRequestPB.Builder pb =
Master.CreateTableRequestPB.newBuilder();
private boolean wait = true;
+ private boolean isPbGenerationDone = false;
/**
* Add a set of hash partitions to the table.
@@ -164,6 +167,23 @@ public class CreateTableOptions {
return this;
}
+ /**
+ * Add range partition with custom hash schema.
+ *
+ * @param rangePartition range partition with custom hash schema
+ * @return this CreateTableOptions object modified accordingly
+ */
+ public CreateTableOptions
addRangePartition(RangePartitionWithCustomHashSchema rangePartition) {
+if (!splitRows.isEmpty()) {
+ throw new IllegalArgumentException(
+ "no range partitions with custom hash schema are allowed when using
" +
+ "split rows to define range partitioning for a table");
+}
+customRangePartitions.add(rangePartition);
+
pb.getPartitionSchemaBuilder().addCustomHashSchemaRanges(rangePartition.toPB());
+return this;
+ }
+
/**
* Add a range partition split. The split row must fall in a ra
[kudu] branch master updated: [java] KUDU-2671 support adding a range with custom hash schema
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 248d9f8ce [java] KUDU-2671 support adding a range with custom hash
schema
248d9f8ce is described below
commit 248d9f8cecd947d700f9e1135db17a30ea1e194a
Author: Alexey Serbin
AuthorDate: Thu Jun 2 17:40:30 2022 -0700
[java] KUDU-2671 support adding a range with custom hash schema
With this patch, Kudu Java client now supports adding a range with
custom hash schema for a table. The patch also contains test cases
to cover the new functionality.
This is a patch to complement [1] at the Kudu Java client side
([1] introduced corresponding changes at the Kudu C++ client).
[1] https://gerrit.cloudera.org/#/c/18663/
Change-Id: Ieaab7a79d4336de7ff6ec84b8c1806407e4fa44e
Reviewed-on: http://gerrit.cloudera.org:8080/18589
Tested-by: Alexey Serbin
Reviewed-by: Mahesh Reddy
Reviewed-by: Attila Bukor
---
.../org/apache/kudu/client/AlterTableOptions.java | 33 +++
.../org/apache/kudu/client/TestAlterTable.java | 262 +++
.../java/org/apache/kudu/client/TestKuduTable.java | 279 -
3 files changed, 567 insertions(+), 7 deletions(-)
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
index 2f01d5798..a2e4f6476 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -363,6 +363,39 @@ public class AlterTableOptions {
return this;
}
+ /**
+ * Similar to the other addRangePartition() methods, but instead of adding a
+ * range with table-wide hash schema, this method adds a range with
+ * custom hash schema.
+ *
+ * @param range the range with custom hash schema
+ * @return this instance
+ */
+ public AlterTableOptions
addRangePartition(RangePartitionWithCustomHashSchema range) {
+Preconditions.checkNotNull(range);
+AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
+step.setType(AlterTableRequestPB.StepType.ADD_RANGE_PARTITION);
+AlterTableRequestPB.AddRangePartition.Builder rangeBuilder =
+AlterTableRequestPB.AddRangePartition.newBuilder();
+rangeBuilder.setRangeBounds(
+new Operation.OperationsEncoder().encodeLowerAndUpperBounds(
+range.getLowerBound(), range.getUpperBound(),
+range.getLowerBoundType(), range.getUpperBoundType()));
+for (org.apache.kudu.Common.PartitionSchemaPB.HashBucketSchemaPB
hashSchema :
+range.toPB().getHashSchemaList()) {
+ Common.PartitionSchemaPB.HashBucketSchemaPB.Builder hbs =
+ rangeBuilder.addCustomHashSchemaBuilder();
+ hbs.mergeFrom(hashSchema);
+}
+step.setAddRangePartition(rangeBuilder);
+if (!pb.hasSchema()) {
+ pb.setSchema(ProtobufHelper.schemaToPb(range.getLowerBound().getSchema(),
+ EnumSet.of(SchemaPBConversionFlags.SCHEMA_PB_WITHOUT_COMMENT,
+ SchemaPBConversionFlags.SCHEMA_PB_WITHOUT_ID)));
+}
+return this;
+ }
+
/**
* Drop the range partition from the table with the specified inclusive
lower bound and exclusive
* upper bound. The bounds must match exactly, and may not span multiple
range partitions.
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
index 377428437..79ddc39c4 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java
@@ -512,6 +512,268 @@ public class TestAlterTable {
assertEquals(100, countRowsInTable(table));
}
+ /**
+ * Test altering a table, adding range partitions with custom hash schema
+ * per range.
+ */
+ @Test(timeout = 10)
+ @KuduTestHarness.MasterServerConfig(flags = {
+ "--enable_per_range_hash_schemas=true",
+ })
+ public void testAlterAddRangeWithCustomHashSchema() throws Exception {
+ArrayList columns = new ArrayList<>(2);
+columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT32)
+.nullable(false)
+.key(true)
+.build());
+columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT32)
+.nullable(false)
+.build());
+final Schema schema = new Schema(columns);
+
+CreateTableOptions createOptions =
+new CreateTableOptions()
+.setRangePartitionColumns(ImmutableList.of("c0"))
+.addHashPartitions(ImmutableList.of("c0"), 2, 0)
+.setNumReplicas(1);
+
+{
+ // Add range partition with the table-wid
[kudu] branch master updated: [java] KUDU-2671 require RANGE_SPECIFIC_HASH_SCHEMA when necessary
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new e696b8a1a [java] KUDU-2671 require RANGE_SPECIFIC_HASH_SCHEMA when
necessary
e696b8a1a is described below
commit e696b8a1a4936d5703a9d5d26469ea997a0b2d90
Author: Alexey Serbin
AuthorDate: Thu Jun 30 20:15:41 2022 -0700
[java] KUDU-2671 require RANGE_SPECIFIC_HASH_SCHEMA when necessary
With this patch, Kudu Java client requires the newly introduced feature
flag RANGE_SPECIFIC_HASH_SCHEMA to be present at the server side when
creating a table with range-specific hash schemas. The flag is also
required when altering already existing table, adding a new range
partition with custom hash schema.
This patch also provides test coverage for the newly introduced
functionality.
This is a follow-up to https://gerrit.cloudera.org/#/c/18633
Change-Id: I7e6625277cc2fb63000999f0c74d9b3d929d5657
Reviewed-on: http://gerrit.cloudera.org:8080/18687
Reviewed-by: Attila Bukor
Tested-by: Alexey Serbin
---
.../org/apache/kudu/client/AlterTableOptions.java | 19 +++-
.../org/apache/kudu/client/AlterTableRequest.java | 8 ++--
.../org/apache/kudu/client/CreateTableOptions.java | 7 ++-
.../org/apache/kudu/client/CreateTableRequest.java | 6 +--
.../org/apache/kudu/client/TestAlterTable.java | 51 ++
.../java/org/apache/kudu/client/TestKuduTable.java | 40 +
6 files changed, 121 insertions(+), 10 deletions(-)
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
index a2e4f6476..1603d0801 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -22,9 +22,11 @@ import static org.apache.kudu.ColumnSchema.Encoding;
import static org.apache.kudu.master.Master.AlterTableRequestPB;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -33,6 +35,7 @@ import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Common;
import org.apache.kudu.Type;
import org.apache.kudu.client.ProtobufHelper.SchemaPBConversionFlags;
+import org.apache.kudu.master.Master;
/**
* This builder must be used to alter a table. At least one change must be
specified.
@@ -42,6 +45,7 @@ import
org.apache.kudu.client.ProtobufHelper.SchemaPBConversionFlags;
public class AlterTableOptions {
private final AlterTableRequestPB.Builder pb =
AlterTableRequestPB.newBuilder();
private boolean wait = true;
+ private boolean isAddingRangeWithCustomHashSchema = false;
/**
* Change a table's name.
@@ -393,6 +397,7 @@ public class AlterTableOptions {
EnumSet.of(SchemaPBConversionFlags.SCHEMA_PB_WITHOUT_COMMENT,
SchemaPBConversionFlags.SCHEMA_PB_WITHOUT_ID)));
}
+isAddingRangeWithCustomHashSchema = true;
return this;
}
@@ -529,4 +534,16 @@ public class AlterTableOptions {
boolean shouldWait() {
return wait;
}
-}
+
+ List getRequiredFeatureFlags() {
+if (!hasAddDropRangePartitions()) {
+ return ImmutableList.of();
+}
+if (!isAddingRangeWithCustomHashSchema) {
+ return
ImmutableList.of(Master.MasterFeatures.RANGE_PARTITION_BOUNDS_VALUE);
+}
+return ImmutableList.of(
+Master.MasterFeatures.RANGE_PARTITION_BOUNDS_VALUE,
+Master.MasterFeatures.RANGE_SPECIFIC_HASH_SCHEMA_VALUE);
+ }
+}
\ No newline at end of file
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
index c323c95e1..a27e01048 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
@@ -42,7 +42,7 @@ class AlterTableRequest extends KuduRpc {
static final String ALTER_TABLE = "AlterTable";
private final String name;
private final AlterTableRequestPB.Builder builder;
- private final List requiredFeatures;
+ private final List featureFlags;
AlterTableRequest(KuduTable masterTable,
String name,
@@ -52,9 +52,7 @@ class AlterTableRequest extends KuduRpc {
super(masterTable, timer, timeoutMillis);
this.name = name;
this.builder = ato.getProtobuf();
-this.requiredFeatures = ato.hasAddDropRangePartitions() ?
-ImmutableList.of(MasterFeatures
