(kudu) branch master updated: KUDU-613: Cleanup of cache code
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 358f0f172 KUDU-613: Cleanup of cache code
358f0f172 is described below
commit 358f0f172609e1a0ea359eb4e8118d0584926b3d
Author: Mahesh Reddy
AuthorDate: Tue Feb 6 15:04:38 2024 -0500
KUDU-613: Cleanup of cache code
This patch moves some classes out of the
anonymous namespace and into the headers
of the cache and nvm_cache files. These
classes will be used by the new SLRU cache.
This path also templatizes the HandleTable class
to be used by both the cache and nvm_cache files.
Change-Id: I506d4577c0ae873b01d7fa4f53846d6fd0f664cf
Reviewed-on: http://gerrit.cloudera.org:8080/21018
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin
---
src/kudu/util/cache.cc | 147 +++-
src/kudu/util/cache.h | 151 +
src/kudu/util/file_cache.cc | 1 +
src/kudu/util/nvm_cache.cc | 162 +++-
src/kudu/util/nvm_cache.h | 32 +
5 files changed, 206 insertions(+), 287 deletions(-)
diff --git a/src/kudu/util/cache.cc b/src/kudu/util/cache.cc
index d141caca7..c0156fe12 100644
--- a/src/kudu/util/cache.cc
+++ b/src/kudu/util/cache.cc
@@ -45,6 +45,7 @@ DEFINE_double(cache_memtracker_approximation_ratio, 0.01,
"this ratio to improve performance. For tests.");
TAG_FLAG(cache_memtracker_approximation_ratio, hidden);
+using RLHandle = kudu::Cache::RLHandle;
using std::atomic;
using std::shared_ptr;
using std::string;
@@ -68,130 +69,6 @@ const Cache::IterationFunc
Cache::kIterateOverAllEntriesFunc = [](
namespace {
-// Recency list cache implementations (FIFO, LRU, etc.)
-
-// Recency list handle. An entry is a variable length heap-allocated structure.
-// Entries are kept in a circular doubly linked list ordered by some recency
-// criterion (e.g., access time for LRU policy, insertion time for FIFO
policy).
-struct RLHandle {
- Cache::EvictionCallback* eviction_callback;
- RLHandle* next_hash;
- RLHandle* next;
- RLHandle* prev;
- size_t charge; // TODO(opt): Only allow uint32_t?
- uint32_t key_length;
- uint32_t val_length;
- std::atomic refs;
- uint32_t hash; // Hash of key(); used for fast sharding and comparisons
-
- // The storage for the key/value pair itself. The data is stored as:
- // [key bytes ...] [padding up to 8-byte boundary] [value bytes ...]
- uint8_t kv_data[1]; // Beginning of key/value pair
-
- Slice key() const {
-return Slice(kv_data, key_length);
- }
-
- uint8_t* mutable_val_ptr() {
-int val_offset = KUDU_ALIGN_UP(key_length, sizeof(void*));
-return &kv_data[val_offset];
- }
-
- const uint8_t* val_ptr() const {
-return const_cast(this)->mutable_val_ptr();
- }
-
- Slice value() const {
-return Slice(val_ptr(), val_length);
- }
-};
-
-// We provide our own simple hash table since it removes a whole bunch
-// of porting hacks and is also faster than some of the built-in hash
-// table implementations in some of the compiler/runtime combinations
-// we have tested. E.g., readrandom speeds up by ~5% over the g++
-// 4.4.3's builtin hashtable.
-class HandleTable {
- public:
- HandleTable() : length_(0), elems_(0), list_(nullptr) { Resize(); }
- ~HandleTable() { delete[] list_; }
-
- RLHandle* Lookup(const Slice& key, uint32_t hash) {
-return *FindPointer(key, hash);
- }
-
- RLHandle* Insert(RLHandle* h) {
-RLHandle** ptr = FindPointer(h->key(), h->hash);
-RLHandle* old = *ptr;
-h->next_hash = (old == nullptr ? nullptr : old->next_hash);
-*ptr = h;
-if (old == nullptr) {
- ++elems_;
- if (elems_ > length_) {
-// Since each cache entry is fairly large, we aim for a small
-// average linked list length (<= 1).
-Resize();
- }
-}
-return old;
- }
-
- RLHandle* Remove(const Slice& key, uint32_t hash) {
-RLHandle** ptr = FindPointer(key, hash);
-RLHandle* result = *ptr;
-if (result != nullptr) {
- *ptr = result->next_hash;
- --elems_;
-}
-return result;
- }
-
- private:
- // The table consists of an array of buckets where each bucket is
- // a linked list of cache entries that hash into the bucket.
- uint32_t length_;
- uint32_t elems_;
- RLHandle** list_;
-
- // Return a pointer to slot that points to a cache entry that
- // matches key/hash. If there is no such cache entry, return a
- // pointer to the trailing slot in the corresponding linked list.
- RLHandle** FindPointer(const Slice& key, uint32_t hash) {
-RLHandle** ptr = &list_[hash & (length_ - 1)];
-while (*ptr != nullptr &&
- ((*ptr)->hash != hash || key != (*ptr)->key())) {
- ptr = &(*ptr)->next_hash;
-}
(kudu) 01/03: [CmakeLists.txt] Fix typo
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit afc3da3ad62dd53d180ef830b2c8d14e2aaf025c
Author: Ádám Bakai
AuthorDate: Fri May 3 12:07:14 2024 +0200
[CmakeLists.txt] Fix typo
Change-Id: I5e8da8f49b201daf99c27cc1fc3793f511e81e6c
Reviewed-on: http://gerrit.cloudera.org:8080/21395
Reviewed-by: Attila Bukor
Reviewed-by: Wang Xixu <1450306...@qq.com>
Tested-by: Attila Bukor
---
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c13d7743a..a8a68bb95 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1332,7 +1332,7 @@ set(KUDU_MIN_TEST_LIBS ${KUDU_BASE_LIBS} kudu_test_main
kudu_test_util)
# Prepend SANITIZER_OPTIONS_OVERRIDE if this is a sanitizer build.
# SANITIZER_OPTIONS_OVERRIDE needs to be linked first so that it is statically
# linked to the test binaries directly. Otherwise the weakly linked default
-# implementations coul be be used when running tests.
+# implementations could be be used when running tests.
if ("${KUDU_USE_ASAN}" OR "${KUDU_USE_TSAN}" OR "${KUDU_USE_UBSAN}")
list(INSERT KUDU_MIN_TEST_LIBS 0 ${SANITIZER_OPTIONS_OVERRIDE})
endif()
(kudu) 02/03: KUDU-3216 fix flakiness in LeadershipChangeOnTskGeneration
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit ca56ee56eb14ea4e91e0ec20b5b8cd857b60423c
Author: Alexey Serbin
AuthorDate: Fri May 3 10:51:18 2024 -0700
KUDU-3216 fix flakiness in LeadershipChangeOnTskGeneration
Before this patch, the LeadershipChangeOnTskGeneration scenario from
CatalogManagerTskITest would fail once in about 50 runs, DEBUG build.
With this patch, the test scenario hasn't had a single failure in
about 300 runs, DEBUG build.
Change-Id: I139e46627206fc13490ca405bb62dc29934dc4be
Reviewed-on: http://gerrit.cloudera.org:8080/21397
Reviewed-by: Wang Xixu <1450306...@qq.com>
Reviewed-by: Attila Bukor
Tested-by: Attila Bukor
---
.../integration-tests/catalog_manager_tsk-itest.cc | 27 ++
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
index cceb1aa07..ec4d1e8c6 100644
--- a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
+++ b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
@@ -124,11 +124,28 @@ class CatalogManagerTskITest : public KuduTest {
auto schema = KuduSchema::FromSchema(CreateKeyValueTestSchema());
unique_ptr table_creator(client->NewTableCreator());
-ASSERT_OK(table_creator->table_name(kTableName)
- .set_range_partition_columns({ "key" })
- .schema(&schema)
- .num_replicas(num_tservers_)
- .Create());
+Status tc_status;
+for (auto i = 0; i < 10; ++i) {
+ // Sometimes, CreateTable requests might arrive when a new system tablet
+ // leader replica hasn't yet replicated NO_OP after establishing its
+ // leadership in a new Raft term. The test is based on
ExternalMiniCluster
+ // where a dedicated API to check for the presence of particular entries
+ // in the WAL is absent. Instead, let's retry CreateTable upon receiving
+ // an error of a particular type and check for the error message: it's
+ // good enough for a test.
+ tc_status = table_creator->table_name(kTableName)
+ .set_range_partition_columns({ "key" })
+ .schema(&schema)
+ .num_replicas(num_tservers_)
+ .Create();
+ if (tc_status.ok()) {
+break;
+ }
+ ASSERT_TRUE(tc_status.IsServiceUnavailable()) << tc_status.ToString();
+ ASSERT_STR_CONTAINS(tc_status.ToString(), "leader is not yet ready");
+ SleepFor(MonoDelta::FromMilliseconds(100));
+}
+ASSERT_OK(tc_status);
// Insert a row.
shared_ptr table;
(kudu) 03/03: [tablet] update the severity level of some tablet metrics
This is an automated email from the ASF dual-hosted git repository. abukor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git commit 35716499d850ed594ea93255efa84708c3d65537 Author: Alexey Serbin AuthorDate: Wed May 1 21:44:34 2024 -0700 [tablet] update the severity level of some tablet metrics The motivation for this update was a request to provide information on the number of rows returned by scanners run against a particular tablet. That's to be on par with the following tablet metrics of the MetricLevel::kInfo severity level: * rows_inserted * rows_updated * rows_upserted * rows_deleted I also took the liberty of updating the level of some other tablet metrics to make their severity level more consistent with the rest of tablet metrics that have similar semantics. Change-Id: If468eeef8488f4a28bdc23a2fc45073e4a3592d4 Reviewed-on: http://gerrit.cloudera.org:8080/21384 Reviewed-by: Attila Bukor Tested-by: Attila Bukor --- src/kudu/tablet/tablet_metrics.cc | 36 ++-- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/kudu/tablet/tablet_metrics.cc b/src/kudu/tablet/tablet_metrics.cc index 71a405490..7e53dde34 100644 --- a/src/kudu/tablet/tablet_metrics.cc +++ b/src/kudu/tablet/tablet_metrics.cc @@ -89,7 +89,7 @@ METRIC_DEFINE_counter(tablet, scanner_rows_returned, "Scanner Rows Returned", "Number of rows returned by scanners to clients. This count " "is measured after predicates are applied, and thus is not " "a reflection of the amount of work being done by scanners.", - kudu::MetricLevel::kDebug); + kudu::MetricLevel::kInfo); METRIC_DEFINE_counter(tablet, scanner_cells_returned, "Scanner Cells Returned", kudu::MetricUnit::kCells, "Number of table cells returned by scanners to clients. This count " @@ -109,7 +109,7 @@ METRIC_DEFINE_counter(tablet, scanner_predicates_disabled, "Scanner Column Predi "This count measures the number of disableable column predicates like " "Bloom filter predicate that are automatically disabled if determined to " "be ineffective.", - kudu::MetricLevel::kInfo); + kudu::MetricLevel::kDebug); METRIC_DEFINE_counter(tablet, scanner_rows_scanned, "Scanner Rows Scanned", kudu::MetricUnit::kRows, @@ -118,7 +118,7 @@ METRIC_DEFINE_counter(tablet, scanner_rows_scanned, "Scanner Rows Scanned", "or MVCC-based filtering. Thus, this is a better measure of actual " "table rows that have been processed by scan operations compared " "to the Scanner Rows Returned metric.", - kudu::MetricLevel::kDebug); + kudu::MetricLevel::kInfo); METRIC_DEFINE_counter(tablet, scanner_cells_scanned_from_disk, "Scanner Cells Scanned From Disk", kudu::MetricUnit::kCells, @@ -147,11 +147,11 @@ METRIC_DEFINE_counter(tablet, scanner_bytes_scanned_from_disk, "Scanner Bytes Sc METRIC_DEFINE_counter(tablet, scans_started, "Scans Started", kudu::MetricUnit::kScanners, "Number of scanners which have been started on this tablet", - kudu::MetricLevel::kDebug); + kudu::MetricLevel::kInfo); METRIC_DEFINE_gauge_size(tablet, tablet_active_scanners, "Active Scanners", kudu::MetricUnit::kScanners, "Number of scanners that are currently active on this tablet", - kudu::MetricLevel::kDebug); + kudu::MetricLevel::kInfo); METRIC_DEFINE_counter(tablet, bloom_lookups, "Bloom Filter Lookups", kudu::MetricUnit::kProbes, @@ -195,7 +195,7 @@ METRIC_DEFINE_counter(tablet, ops_timed_out_in_prepare_queue, "corresponding operations were waiting in the tablet's " "prepare queue, and thus were not started but " "acknowledged with TimedOut error status.", - kudu::MetricLevel::kInfo); + kudu::MetricLevel::kWarn); METRIC_DEFINE_histogram(tablet, bloom_lookups_per_op, "Bloom Lookups per Operation", kudu::MetricUnit::kProbes, @@ -221,7 +221,7 @@ METRIC_DEFINE_histogram(tablet, delta_file_lookups_per_op, "Delta File Lookups p "operation. A single operation may perform several delta file " "lookups if the tablet is not fully compacted. High frequency of " "high values may indicate that compaction is falling
(kudu) branch master updated (b72fc6255 -> 35716499d)
This is an automated email from the ASF dual-hosted git repository. abukor pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git from b72fc6255 [metrics] Add metrics for create and delete op time new afc3da3ad [CmakeLists.txt] Fix typo new ca56ee56e KUDU-3216 fix flakiness in LeadershipChangeOnTskGeneration new 35716499d [tablet] update the severity level of some tablet metrics The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CMakeLists.txt | 2 +- .../integration-tests/catalog_manager_tsk-itest.cc | 27 +--- src/kudu/tablet/tablet_metrics.cc | 36 +++--- 3 files changed, 41 insertions(+), 24 deletions(-)
