[kudu] branch master updated: [Tools] Rebuild master according to part of tables not all tables
This is an automated email from the ASF dual-hosted git repository.
zhangyifan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 81f7c5b43 [Tools] Rebuild master according to part of tables not all
tables
81f7c5b43 is described below
commit 81f7c5b43fe4546ee1b7526b51a13126da706980
Author: xinghuayu007 <1450306...@qq.com>
AuthorDate: Thu May 26 09:52:28 2022 +0800
[Tools] Rebuild master according to part of tables not all tables
Current function unsafe_rebuild will get all tables's meta data
from tservers, and rebuild a new sys table. The cost is very
high when only part of tables got damaged.
This patch will rebuild master with configured tables. That
means if some tables meta data get damaged, it only needs to
repair these damaged tables meta data.
Change-Id: I45d812c8a16228ac937603872161969eb05136eb
Reviewed-on: http://gerrit.cloudera.org:8080/18572
Reviewed-by: Yifan Zhang
Tested-by: Yifan Zhang
---
src/kudu/master/sys_catalog-test.cc | 45 +-
src/kudu/master/sys_catalog.cc | 30 +++
src/kudu/master/sys_catalog.h| 27 ++
src/kudu/tools/kudu-admin-test.cc| 159 ---
src/kudu/tools/master_rebuilder.cc | 148 +++-
src/kudu/tools/master_rebuilder.h| 6 +-
src/kudu/tools/tool_action_master.cc | 7 +-
7 files changed, 342 insertions(+), 80 deletions(-)
diff --git a/src/kudu/master/sys_catalog-test.cc
b/src/kudu/master/sys_catalog-test.cc
index 67957c7cb..e28a10625 100644
--- a/src/kudu/master/sys_catalog-test.cc
+++ b/src/kudu/master/sys_catalog-test.cc
@@ -95,26 +95,6 @@ class SysCatalogTest : public KuduTest {
unique_ptr proxy_;
};
-class TestTableLoader : public TableVisitor {
- public:
- void Reset() {
-tables.clear();
- }
-
- Status VisitTable(const string& table_id,
-const SysTablesEntryPB& metadata) override {
-// Setup the table info
-scoped_refptr table = new TableInfo(table_id);
-TableMetadataLock l(table.get(), LockMode::WRITE);
-l.mutable_data()->pb.CopyFrom(metadata);
-l.Commit();
-tables.emplace_back(std::move(table));
-return Status::OK();
- }
-
- vector> tables;
-};
-
static bool PbEquals(const google::protobuf::Message& a, const
google::protobuf::Message& b) {
return pb_util::SecureDebugString(a) == pb_util::SecureDebugString(b);
}
@@ -130,7 +110,7 @@ static bool MetadatasEqual(const scoped_refptr& ti_a,
// Test the sys-catalog tables basic operations (add, update, delete,
// visit)
TEST_F(SysCatalogTest, TestSysCatalogTablesOperations) {
- TestTableLoader loader;
+ TableInfoLoader loader;
auto* sys_catalog = master_->catalog_manager()->sys_catalog();
ASSERT_OK(sys_catalog->VisitTables(&loader));
@@ -226,27 +206,6 @@ TEST_F(SysCatalogTest, TestTableInfoCommit) {
}
}
-class TestTabletLoader : public TabletVisitor {
- public:
- void Reset() {
-tablets.clear();
- }
-
- Status VisitTablet(const string& /*table_id*/,
- const string& tablet_id,
- const SysTabletsEntryPB& metadata) override {
-// Setup the tablet info
-scoped_refptr tablet = new TabletInfo(nullptr, tablet_id);
-TabletMetadataLock l(tablet.get(), LockMode::WRITE);
-l.mutable_data()->pb.CopyFrom(metadata);
-l.Commit();
-tablets.emplace_back(std::move(tablet));
-return Status::OK();
- }
-
- vector> tablets;
-};
-
// Create a new TabletInfo. The object is in uncommitted
// state.
static scoped_refptr CreateTablet(
@@ -274,7 +233,7 @@ TEST_F(SysCatalogTest, TestSysCatalogTabletsOperations) {
SysCatalogTable* sys_catalog = master_->catalog_manager()->sys_catalog();
- TestTabletLoader loader;
+ TabletInfoLoader loader;
ASSERT_OK(master_->catalog_manager()->sys_catalog()->VisitTablets(&loader));
ASSERT_EQ(0, loader.tablets.size());
diff --git a/src/kudu/master/sys_catalog.cc b/src/kudu/master/sys_catalog.cc
index 58a44922b..1c346cacf 100644
--- a/src/kudu/master/sys_catalog.cc
+++ b/src/kudu/master/sys_catalog.cc
@@ -1157,5 +1157,35 @@ void SysCatalogTable::InitLocalRaftPeerPB() {
*local_peer_pb_.mutable_last_known_addr() = HostPortToPB(hps[0]);
}
+void TableInfoLoader::Reset() {
+ tables.clear();
+}
+
+Status TableInfoLoader::VisitTable(const string& table_id,
+ const SysTablesEntryPB& metadata) {
+ // Setup the table info
+ scoped_refptr table = new TableInfo(table_id);
+ TableMetadataLock l(table.get(), LockMode::WRITE);
+ l.mutable_data()->pb.CopyFrom(metadata);
+ l.Commit();
+ tables.emplace_back(std::move(table));
+ return Status::OK();
+}
+
+void TabletInfoLoader::Reset() {
+ tablets.clear();
+}
+
+Status TabletInfoLoader::VisitTablet(const string& /*table_id*/,
+ const string
[kudu] branch master updated: KUDU-2671 add tests to check for number of hash buckets
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 55d711d78 KUDU-2671 add tests to check for number of hash buckets
55d711d78 is described below
commit 55d711d78a21c76718b06dede79de2e9b9d5e8e8
Author: Alexey Serbin
AuthorDate: Fri Jul 15 11:43:00 2022 -0700
KUDU-2671 add tests to check for number of hash buckets
This patch adds a few test scenarios to cover the handling of invalid
number of hash buckets for range-specific hash schemas in AlterTable
RPCs.
Change-Id: I6bdd728b5dea7fa864648e167a1a76b07c706e8f
Reviewed-on: http://gerrit.cloudera.org:8080/18739
Tested-by: Alexey Serbin
Reviewed-by: Alexey Serbin
---
.../java/org/apache/kudu/client/TestKuduTable.java | 49 +++
src/kudu/client/flex_partitioning_client-test.cc | 47 +++
src/kudu/master/master-test.cc | 95 ++
3 files changed, 191 insertions(+)
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
index 80101d806..28c4ac66c 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
@@ -1700,6 +1700,55 @@ public class TestKuduTable {
}
}
+ @Test(timeout = 10)
+ public void testAlterTableAddRangeCustomHashSchemaWrongBucketsNumber()
throws Exception {
+final List columns = ImmutableList.of(
+new ColumnSchema.ColumnSchemaBuilder("key",
Type.INT32).key(true).build(),
+new ColumnSchema.ColumnSchemaBuilder("value", Type.STRING).build());
+final Schema schema = new Schema(columns);
+
+CreateTableOptions options = getBasicCreateTableOptions();
+// Add table-wide schema for the table.
+options.addHashPartitions(ImmutableList.of("key"), 2, 0);
+// Add a range partition with table-wide hash schema.
+{
+ PartialRow lower = schema.newPartialRow();
+ lower.addInt(0, -100);
+ PartialRow upper = schema.newPartialRow();
+ upper.addInt(0, 0);
+ options.addRangePartition(lower, upper);
+}
+
+client.createTable(tableName, schema, options);
+
+PartialRow lower = schema.newPartialRow();
+lower.addInt(0, 0);
+PartialRow upper = schema.newPartialRow();
+upper.addInt(0, 100);
+
+// Try to add range with a single hash bucket -- it should not be possible.
+for (int hashBucketNum = -1; hashBucketNum < 2; ++hashBucketNum) {
+ try {
+RangePartitionWithCustomHashSchema range =
+new RangePartitionWithCustomHashSchema(
+lower,
+upper,
+RangePartitionBound.INCLUSIVE_BOUND,
+RangePartitionBound.EXCLUSIVE_BOUND);
+range.addHashPartitions(ImmutableList.of("key"), hashBucketNum, 0);
+
+client.alterTable(tableName, new
AlterTableOptions().addRangePartition(range));
+fail(String.format("should not be able to add a partition with " +
+"invalid range-specific hash schema of %d hash buckets",
hashBucketNum));
+ } catch (KuduException ex) {
+final String errmsg = ex.getMessage();
+assertTrue(errmsg, ex.getStatus().isInvalidArgument());
+assertTrue(String.format("%d hash buckets: %s", hashBucketNum, errmsg),
+errmsg.matches("must have at least two hash buckets"));
+ }
+}
+ }
+
@Test(timeout = 10)
@KuduTestHarness.MasterServerConfig(flags = {
"--enable_per_range_hash_schemas=false",
diff --git a/src/kudu/client/flex_partitioning_client-test.cc
b/src/kudu/client/flex_partitioning_client-test.cc
index 77e67d990..1d3909940 100644
--- a/src/kudu/client/flex_partitioning_client-test.cc
+++ b/src/kudu/client/flex_partitioning_client-test.cc
@@ -2036,5 +2036,52 @@ TEST_F(FlexPartitioningScanTest, MaxKeyValue) {
}
}
+// Try adding range partition with custom hash schema where the number of
+// hash buckets is invalid.
+TEST_F(FlexPartitioningAlterTableTest, AddRangeWithWrongHashBucketsNumber) {
+ constexpr const char* const kCol0 = "c0";
+ constexpr const char* const kCol1 = "c1";
+ constexpr const char* const kErrMsg =
+ "at least two buckets are required to establish hash partitioning";
+ constexpr const char* const kTableName =
"AddRangeWithWrongHashBucketsNumber";
+
+ KuduSchema schema;
+ {
+KuduSchemaBuilder b;
+b.AddColumn(kCol0)->Type(KuduColumnSchema::INT32)->NotNull()->PrimaryKey();
+b.AddColumn(kCol1)->Type(KuduColumnSchema::STRING)->Nullable();
+ASSERT_OK(b.Build(&schema));
+ }
+
+ unique_ptr table_creator(client_->NewTableCreator());
+ table_creator->table_name(kTableName)
+ .schema(&schema)
+ .num_replicas(1)
+ .add_ha
[kudu] branch master updated: [client] KUDU-2671 move KuduRangePartition out of KuduTableCreator
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 6fdd91905 [client] KUDU-2671 move KuduRangePartition out of
KuduTableCreator
6fdd91905 is described below
commit 6fdd9190592fe47c2202e75edd2773832bf29968
Author: Alexey Serbin
AuthorDate: Wed Jul 20 17:27:40 2022 -0700
[client] KUDU-2671 move KuduRangePartition out of KuduTableCreator
This patch moves the KuduRangePartition class out of the
KuduTableCreator class. The reasoning behind this change is two-fold:
* make the API of KuduTableAlterer more consistent with the added
AddRangePartition(KuduRangePartition* partition) method
* avoid issues with embedded classes while cythonizing C++ client API
while adding support for range-specific hash schemas
This change might break ABI compatibility, but we haven't announced
support for range-specific hash schemas in Kudu 1.16 since it was not
ready yet. Any Kudu C++ client application that started using that
experimental API (which is very unlikely) should be recompiled with
updated headers and the kudu_client library.
Change-Id: I5af14e7e802baca496e13e05860d66685914dd29
Reviewed-on: http://gerrit.cloudera.org:8080/18765
Reviewed-by: Abhishek Chennaka
Reviewed-by: Mahesh Reddy
Tested-by: Alexey Serbin
Reviewed-by: Attila Bukor
---
src/kudu/client/client.cc| 32 +++---
src/kudu/client/client.h | 131 ---
src/kudu/client/flex_partitioning_client-test.cc | 12 +--
src/kudu/client/scan_token-test.cc | 18 ++--
src/kudu/client/table_alterer-internal.h | 2 +-
src/kudu/client/table_creator-internal.cc| 8 +-
src/kudu/client/table_creator-internal.h | 10 +-
7 files changed, 103 insertions(+), 110 deletions(-)
diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc
index 7adc33800..d31214371 100644
--- a/src/kudu/client/client.cc
+++ b/src/kudu/client/client.cc
@@ -1048,28 +1048,26 @@ Status KuduTableCreator::Create() {
return Status::OK();
}
-KuduTableCreator::KuduRangePartition::KuduRangePartition(
+KuduRangePartition::KuduRangePartition(
KuduPartialRow* lower_bound,
KuduPartialRow* upper_bound,
-RangePartitionBound lower_bound_type,
-RangePartitionBound upper_bound_type)
+KuduTableCreator::RangePartitionBound lower_bound_type,
+KuduTableCreator::RangePartitionBound upper_bound_type)
: data_(new Data(lower_bound, upper_bound, lower_bound_type,
upper_bound_type)) {
}
-KuduTableCreator::KuduRangePartition::~KuduRangePartition() {
+KuduRangePartition::~KuduRangePartition() {
delete data_;
}
-Status KuduTableCreator::KuduRangePartition::add_hash_partitions(
+Status KuduRangePartition::add_hash_partitions(
const vector& columns,
int32_t num_buckets,
int32_t seed) {
if (seed < 0) {
-// TODO(aserbin): change the signature of
-//KuduRangePartition::add_hash_partitions() to use uint32_t
-//for the 'seed' parameter while it's still possible since
-//the client API hasn't been released yet
-return Status::InvalidArgument("hash seed must non-negative");
+// int32_t, not uint32_t for seed is used to be "compatible" with the type
+// of the 'seed' parameter for KuduTableCreator::add_hash_partitions().
+return Status::InvalidArgument("hash seed must be non-negative");
}
return data_->add_hash_partitions(columns, num_buckets, seed);
}
@@ -1543,9 +1541,8 @@ KuduTableAlterer*
KuduTableAlterer::AddRangePartitionWithDimension(
Data::Step s { AlterTableRequestPB::ADD_RANGE_PARTITION,
nullptr,
- std::unique_ptr(
- new KuduTableCreator::KuduRangePartition(
- lower_bound, upper_bound, lower_bound_type,
upper_bound_type)),
+ std::unique_ptr(new KuduRangePartition(
+ lower_bound, upper_bound, lower_bound_type,
upper_bound_type)),
dimension_label.empty() ? nullopt :
make_optional(dimension_label) };
data_->steps_.emplace_back(std::move(s));
data_->has_alter_partitioning_steps = true;
@@ -1553,7 +1550,7 @@ KuduTableAlterer*
KuduTableAlterer::AddRangePartitionWithDimension(
}
KuduTableAlterer* KuduTableAlterer::AddRangePartition(
-KuduTableCreator::KuduRangePartition* partition) {
+KuduRangePartition* partition) {
CHECK(partition);
if (partition->data_->lower_bound_ == nullptr ||
partition->data_->upper_bound_ == nullptr) {
data_->status_ = Status::InvalidArgument("range partition bounds may not
be null");
@@ -1572,7 +1569,7 @@ KuduTableAlterer* KuduTableAlterer::AddRangePartition(
Data::Step s {
[kudu] branch master updated: [client] remove C++11 extensions from exported header
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 9aa17addd [client] remove C++11 extensions from exported header
9aa17addd is described below
commit 9aa17addd132956a2a09bf522aa5d565a4098775
Author: Alexey Serbin
AuthorDate: Wed Jul 20 15:56:51 2022 -0700
[client] remove C++11 extensions from exported header
Changelist 708fdff19 included updates to the exported C++ header
write_op.h. The C++ client API is still kept C++98 compliant,
so C++11 extensions should not be present.
This patch reverts the changes in src/kudu/client/write_op.h
that 708fdff19 introduced.
This is a follow-up to 708fdff19073dbe9dc4eb9492551443aa38111f9.
Change-Id: I042ccaa76e424475dd2347505a611cdcab5c21d3
Reviewed-on: http://gerrit.cloudera.org:8080/18764
Tested-by: Alexey Serbin
Reviewed-by: Attila Bukor
---
src/kudu/client/write_op.h | 43 ++-
1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/src/kudu/client/write_op.h b/src/kudu/client/write_op.h
index e5b76488e..a1e9e30a6 100644
--- a/src/kudu/client/write_op.h
+++ b/src/kudu/client/write_op.h
@@ -30,6 +30,7 @@
#ifdef KUDU_HEADERS_NO_STUBS
#include "kudu/gutil/macros.h"
+#include "kudu/gutil/port.h"
#else
#include "kudu/client/stubs.h"
#endif
@@ -140,16 +141,16 @@ class KUDU_EXPORT KuduWriteOperation {
/// columns which do not have default values.
class KUDU_EXPORT KuduInsert : public KuduWriteOperation {
public:
- ~KuduInsert() override;
+ virtual ~KuduInsert();
/// @copydoc KuduWriteOperation::ToString()
- std::string ToString() const override { return "INSERT " + row_.ToString(); }
+ virtual std::string ToString() const OVERRIDE { return "INSERT " +
row_.ToString(); }
protected:
/// @cond PROTECTED_MEMBERS_DOCUMENTED
/// @copydoc KuduWriteOperation::type()
- Type type() const override {
+ virtual Type type() const OVERRIDE {
return INSERT;
}
@@ -167,16 +168,16 @@ class KUDU_EXPORT KuduInsert : public KuduWriteOperation {
/// columns which do not have default values.
class KUDU_EXPORT KuduInsertIgnore : public KuduWriteOperation {
public:
- ~KuduInsertIgnore() override;
+ virtual ~KuduInsertIgnore();
/// @copydoc KuduWriteOperation::ToString()
- std::string ToString() const override { return "INSERT IGNORE " +
row_.ToString(); }
+ virtual std::string ToString() const OVERRIDE { return "INSERT IGNORE " +
row_.ToString(); }
protected:
/// @cond PROTECTED_MEMBERS_DOCUMENTED
/// @copydoc KuduWriteOperation::type()
- Type type() const override {
+ virtual Type type() const OVERRIDE {
return INSERT_IGNORE;
}
@@ -193,16 +194,16 @@ class KUDU_EXPORT KuduInsertIgnore : public
KuduWriteOperation {
/// See KuduInsert for more details.
class KUDU_EXPORT KuduUpsert : public KuduWriteOperation {
public:
- ~KuduUpsert() override;
+ virtual ~KuduUpsert();
/// @copydoc KuduWriteOperation::ToString()
- std::string ToString() const override { return "UPSERT " + row_.ToString(); }
+ virtual std::string ToString() const OVERRIDE { return "UPSERT " +
row_.ToString(); }
protected:
/// @cond PROTECTED_MEMBERS_DOCUMENTED
/// @copydoc KuduWriteOperation::type()
- Type type() const override {
+ virtual Type type() const OVERRIDE {
return UPSERT;
}
@@ -220,16 +221,16 @@ class KUDU_EXPORT KuduUpsert : public KuduWriteOperation {
/// in the schema to be set in the embedded KuduPartialRow object.
class KUDU_EXPORT KuduUpdate : public KuduWriteOperation {
public:
- ~KuduUpdate() override;
+ virtual ~KuduUpdate();
/// @copydoc KuduWriteOperation::ToString()
- std::string ToString() const override { return "UPDATE " + row_.ToString(); }
+ virtual std::string ToString() const OVERRIDE { return "UPDATE " +
row_.ToString(); }
protected:
/// @cond PROTECTED_MEMBERS_DOCUMENTED
/// @copydoc KuduWriteOperation::type()
- Type type() const override {
+ virtual Type type() const OVERRIDE {
return UPDATE;
}
@@ -246,16 +247,16 @@ class KUDU_EXPORT KuduUpdate : public KuduWriteOperation {
/// in the schema to be set in the embedded KuduPartialRow object.
class KUDU_EXPORT KuduUpdateIgnore : public KuduWriteOperation {
public:
- ~KuduUpdateIgnore() override;
+ virtual ~KuduUpdateIgnore();
/// @copydoc KuduWriteOperation::ToString()
- std::string ToString() const override { return "UPDATE IGNORE " +
row_.ToString(); }
+ virtual std::string ToString() const OVERRIDE { return "UPDATE IGNORE " +
row_.ToString(); }
protected:
/// @cond PROTECTED_MEMBERS_DOCUMENTED
/// @copydoc KuduWriteOperation::type()
- Type type() const override {
+ virtual Type type() const OVERRIDE {
return UPDATE_IGNORE;
}
@@ -272,16 +273
[kudu] branch master updated: [ranger-kms] part1: thirdparty changes
This is an automated email from the ASF dual-hosted git repository. abukor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git The following commit(s) were added to refs/heads/master by this push: new 54ee2103d [ranger-kms] part1: thirdparty changes 54ee2103d is described below commit 54ee2103d9dde5347891d4cf4a4dbd674774f475 Author: Zoltan Chovan AuthorDate: Tue May 3 20:46:23 2022 +0200 [ranger-kms] part1: thirdparty changes Added RangerKMS to thirdparty libs, to facilitate Ranger KMS integration in following changes. Change-Id: Id46312bdb81e2fed5a1f19be17df5b974a6716a0 Reviewed-on: http://gerrit.cloudera.org:8080/18644 Reviewed-by: Khazar Mammadli Reviewed-by: Attila Bukor Tested-by: Attila Bukor --- thirdparty/build-thirdparty.sh| 15 ++- thirdparty/download-thirdparty.sh | 6 ++ thirdparty/vars.sh| 4 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index e7ab77256..7b95460c9 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -107,7 +107,8 @@ else "ranger") F_RANGER=1 ;; "oatpp")F_OATPP=1 ;; "oatpp-swagger") F_OATPP_SWAGGER=1 ;; - "jwt-cpp") F_JWT_CPP=1;; + "jwt-cpp") F_JWT_CPP=1 ;; + "ranger-kms") F_RANGER_KMS=1 ;; *) echo "Unknown module: $arg"; exit 1 ;; esac done @@ -286,6 +287,18 @@ if [ -n "$F_COMMON" -o -n "$F_RANGER" ]; then $PREFIX/opt/ranger/ews/webapp/WEB-INF/classes/conf fi +if [ -n "$F_COMMON" -o -n "$F_RANGER_KMS" ]; then + mkdir -p $PREFIX/opt + # Remove any hadoop jars included in the Ranger package to avoid unexpected + # runtime behavior due to different versions of hadoop jars. + rm -rf $RANGER_KMS_SOURCE/lib/hadoop-[a-z-]*.jar + ln -nsf $RANGER_KMS_SOURCE $PREFIX/opt/ranger-kms + + # Symlink conf.dist to conf + ln -nsf $PREFIX/opt/ranger-kms/ews/webapp/WEB-INF/classes/conf.dist \ + $PREFIX/opt/ranger-kms/ews/webapp/WEB-INF/classes/conf +fi + ### Build C dependencies without instrumentation PREFIX=$PREFIX_DEPS diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index e7bf40071..e9606fcc5 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -485,5 +485,11 @@ fetch_and_patch \ $JWT_CPP_SOURCE \ $JWT_CPP_PATCHLEVEL +RANGER_KMS_PATCHLEVEL=0 +fetch_and_patch \ + $RANGER_KMS_NAME.tar.gz \ + $RANGER_KMS_SOURCE \ + $RANGER_KMS_PATCHLEVEL + echo "---" echo "Thirdparty dependencies downloaded successfully" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 4e94419f3..a5971670a 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -259,6 +259,10 @@ RANGER_VERSION=2.1.0 RANGER_NAME=ranger-$RANGER_VERSION-admin RANGER_SOURCE=$TP_SOURCE_DIR/$RANGER_NAME +RANGER_KMS_VERSION=2.1.0 # this probably should match the ranger version +RANGER_KMS_NAME=ranger-$RANGER_KMS_VERSION-kms +RANGER_KMS_SOURCE=$TP_SOURCE_DIR/$RANGER_KMS_NAME + OATPP_VERSION=1.2.5 OATPP_NAME=oatpp-$OATPP_VERSION OATPP_SOURCE=$TP_SOURCE_DIR/$OATPP_NAME
[kudu] branch master updated: KUDU-3384: Handling IncrementEncodedKey failure
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 1bde51fa0 KUDU-3384: Handling IncrementEncodedKey failure
1bde51fa0 is described below
commit 1bde51fa0d5800ee76160258a8e879f7cce66a75
Author: zhangyifan27
AuthorDate: Wed Jul 20 14:08:28 2022 +0800
KUDU-3384: Handling IncrementEncodedKey failure
While using lower/upper of a DRS to optimize scan at DRS level, we
should handle the cases where a upper bound key cannot be incremented.
This patch fixes it, enables the reproduction scenario and also adds
a regression test.
Change-Id: I22c48f9893364c3fad5597431e4acfbcb2a4b43d
Reviewed-on: http://gerrit.cloudera.org:8080/18761
Tested-by: Alexey Serbin
Reviewed-by: Alexey Serbin
---
src/kudu/client/flex_partitioning_client-test.cc | 4 +-
src/kudu/tablet/cfile_set-test.cc| 55 +++-
src/kudu/tablet/cfile_set.cc | 8 +++-
3 files changed, 61 insertions(+), 6 deletions(-)
diff --git a/src/kudu/client/flex_partitioning_client-test.cc
b/src/kudu/client/flex_partitioning_client-test.cc
index e23eb7272..c9d65ca72 100644
--- a/src/kudu/client/flex_partitioning_client-test.cc
+++ b/src/kudu/client/flex_partitioning_client-test.cc
@@ -1970,9 +1970,7 @@ class FlexPartitioningScanTest : public
FlexPartitioningTest {
};
// This scenario is to reproduce the issue described in KUDU-3384.
-//
-// TODO(aserbin): enable the scenario once KUDU-3384 is fixed
-TEST_F(FlexPartitioningScanTest, DISABLED_MaxKeyValue) {
+TEST_F(FlexPartitioningScanTest, MaxKeyValue) {
static constexpr const char* const kTableName = "max_key_value";
unique_ptr table_creator(client_->NewTableCreator());
diff --git a/src/kudu/tablet/cfile_set-test.cc
b/src/kudu/tablet/cfile_set-test.cc
index 6b115cbe9..0e6723d87 100644
--- a/src/kudu/tablet/cfile_set-test.cc
+++ b/src/kudu/tablet/cfile_set-test.cc
@@ -100,7 +100,7 @@ class TestCFileSet : public KuduRowSetTest {
FLAGS_cfile_default_block_size = 512;
}
- // Write out a test rowset with two int columns.
+ // Write out a test rowset with three int columns.
// The first column contains the row index * 2.
// The second contains the row index * 10.
// The third column contains index * 100, but is never read.
@@ -121,6 +121,30 @@ class TestCFileSet : public KuduRowSetTest {
ASSERT_OK(rsw.Finish());
}
+ void WriteTestRowSetWithMaxValue(int nrows) {
+DiskRowSetWriter rsw(
+rowset_meta_.get(), &schema_, BloomFilterSizing::BySizeAndFPRate(32 *
1024, 0.01F));
+
+ASSERT_OK(rsw.Open());
+
+RowBuilder rb(&schema_);
+int i = INT32_MAX - nrows + 1;
+while (i != INT32_MAX) {
+ rb.Reset();
+ rb.AddInt32(i);
+ rb.AddInt32(i);
+ rb.AddInt32(i);
+ ASSERT_OK_FAST(WriteRow(rb.data(), &rsw));
+ i++;
+}
+rb.Reset();
+rb.AddInt32(i);
+rb.AddInt32(i);
+rb.AddInt32(i);
+ASSERT_OK_FAST(WriteRow(rb.data(), &rsw));
+ASSERT_OK(rsw.Finish());
+ }
+
// Int32 type add probe to the bloom filter.
// bf1_contain: 0 2 4 6 8 ... (2n)th key for column 1 to form bloom filter.
// bf1_exclude: 1 3 5 7 9 ... (2n + 1)th key for column 1 to form bloom
filter.
@@ -628,6 +652,35 @@ TEST_F(TestCFileSet, TestInListPredicates) {
DoTestInListScan(fileset, 1000, 10);
}
+// Regression test for KUDU-3384
+TEST_F(TestCFileSet, TestKUDU3384) {
+ constexpr int kNumRows = 10;
+ WriteTestRowSetWithMaxValue(kNumRows);
+
+ shared_ptr fileset;
+ ASSERT_OK(CFileSet::Open(
+ rowset_meta_, MemTracker::GetRootTracker(),
MemTracker::GetRootTracker(), nullptr, &fileset));
+
+ // Create iterator.
+ unique_ptr cfile_iter(fileset->NewIterator(&schema_,
nullptr));
+ unique_ptr
iter(NewMaterializingIterator(std::move(cfile_iter)));
+
+ // Check a full scan is successful.
+ ScanSpec spec;
+ ASSERT_OK(iter->Init(&spec));
+ RowBlockMemory mem(1024);
+ RowBlock block(&schema_, 100, &mem);
+ int selected_size = 0;
+ while (iter->HasNext()) {
+mem.Reset();
+ASSERT_OK_FAST(iter->NextBlock(&block));
+selected_size += block.selection_vector()->CountSelected();
+ }
+ ASSERT_EQ(kNumRows, selected_size);
+ // Check a range scan is successful.
+ DoTestRangeScan(fileset, INT32_MAX - kNumRows, INT32_MAX);
+}
+
class InListPredicateBenchmark : public KuduRowSetTest {
public:
InListPredicateBenchmark()
diff --git a/src/kudu/tablet/cfile_set.cc b/src/kudu/tablet/cfile_set.cc
index e145edbae..7620eea51 100644
--- a/src/kudu/tablet/cfile_set.cc
+++ b/src/kudu/tablet/cfile_set.cc
@@ -442,8 +442,12 @@ Status CFileSet::Iterator::OptimizePKPredicates(ScanSpec*
spec) {
RETURN_NOT_OK(EncodedKey::DecodeEncodedString(
tablet_schema, &arena_, base_data_->max_encoded_key_, &implicit_ub_key));
- RETURN_NOT
