This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new b38ef432eb Add a dir-entry setting to volumes.config (#11838)
b38ef432eb is described below
commit b38ef432ebf9910f33258f88ed446627c2b333f7
Author: Leif Hedstrom <[email protected]>
AuthorDate: Thu Nov 14 16:48:14 2024 -0700
Add a dir-entry setting to volumes.config (#11838)
---
doc/admin-guide/files/volume.config.en.rst | 11 ++++++++++-
src/iocore/cache/Cache.cc | 2 +-
src/iocore/cache/CacheHosting.cc | 12 +++++++++++-
src/iocore/cache/CacheProcessor.cc | 1 +
src/iocore/cache/P_CacheHosting.h | 1 +
src/iocore/cache/Stripe.cc | 20 ++++++++++++--------
src/iocore/cache/Stripe.h | 7 ++++---
src/iocore/cache/StripeSM.cc | 4 ++--
src/iocore/cache/StripeSM.h | 2 +-
9 files changed, 43 insertions(+), 17 deletions(-)
diff --git a/doc/admin-guide/files/volume.config.en.rst
b/doc/admin-guide/files/volume.config.en.rst
index 7aa0426d3f..91fc2e648a 100644
--- a/doc/admin-guide/files/volume.config.en.rst
+++ b/doc/admin-guide/files/volume.config.en.rst
@@ -68,6 +68,15 @@ sits in front of a volume. This may be desirable if you are
using something lik
ramdisks, to avoid wasting RAM and cpu time on double caching objects.
+Optional directory entry sizing
+-------------------------------
+
+You can also add an option ``avg_obj_size=<size>`` to the volume configuration
+line. This overrides the global
:ts:cv:`proxy.config.cache.min_average_object_size`
+configuration for this volume. This is useful if you have a volume that is
dedicated
+for say very small objects, and you need a lot of directory entries to store
them.
+
+
Exclusive spans and volume sizes
================================
@@ -106,5 +115,5 @@ ramcache has been disabled.::
volume=1 scheme=http size=20%
volume=2 scheme=http size=20%
volume=3 scheme=http size=20%
- volume=4 scheme=http size=20%
+ volume=4 scheme=http size=20% avg_obj_size=4096
volume=5 scheme=http size=20% ramcache=false
diff --git a/src/iocore/cache/Cache.cc b/src/iocore/cache/Cache.cc
index 6e80427aab..04f88f3416 100644
--- a/src/iocore/cache/Cache.cc
+++ b/src/iocore/cache/Cache.cc
@@ -273,7 +273,7 @@ Cache::open(bool clear, bool /* fix ATS_UNUSED */)
for (; q; q = q->link.next) {
blocks = q->b->len;
CacheDisk *d = cp->disk_stripes[i]->disk;
- cp->stripes[vol_no] = new StripeSM(d, blocks,
q->b->offset);
+ cp->stripes[vol_no] = new StripeSM(d, blocks,
q->b->offset, cp->avg_obj_size);
cp->stripes[vol_no]->cache = this;
cp->stripes[vol_no]->cache_vol = cp;
diff --git a/src/iocore/cache/CacheHosting.cc b/src/iocore/cache/CacheHosting.cc
index cc1daf61d4..e984d41f0b 100644
--- a/src/iocore/cache/CacheHosting.cc
+++ b/src/iocore/cache/CacheHosting.cc
@@ -654,6 +654,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
int size = 0;
int in_percent = 0;
bool ramcache_enabled = true;
+ int avg_obj_size = -1; // Defaults
while (true) {
// skip all blank spaces at beginning of line
@@ -741,6 +742,13 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
} else {
in_percent = 0;
}
+ } else if (strcasecmp(tmp, "avg_obj_size") == 0) { // match avg_obj_size
+ tmp += 13;
+ avg_obj_size = atoi(tmp);
+
+ while (ParseRules::is_digit(*tmp)) {
+ tmp++;
+ }
} else if (strcasecmp(tmp, "ramcache") == 0) { // match ramcache
tmp += 9;
if (!strcasecmp(tmp, "false")) {
@@ -767,7 +775,8 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
/* add the config */
ConfigVol *configp = new ConfigVol();
- configp->number = volume_number;
+
+ configp->number = volume_number;
if (in_percent) {
configp->percent = size;
configp->in_percent = true;
@@ -776,6 +785,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
}
configp->scheme = scheme;
configp->size = size;
+ configp->avg_obj_size = avg_obj_size;
configp->cachep = nullptr;
configp->ramcache_enabled = ramcache_enabled;
cp_queue.enqueue(configp);
diff --git a/src/iocore/cache/CacheProcessor.cc
b/src/iocore/cache/CacheProcessor.cc
index 916f4a66e1..402fc619c5 100644
--- a/src/iocore/cache/CacheProcessor.cc
+++ b/src/iocore/cache/CacheProcessor.cc
@@ -1177,6 +1177,7 @@ cplist_update()
if (config_vol->number == cp->vol_number) {
if (cp->scheme == config_vol->scheme) {
cp->ramcache_enabled = config_vol->ramcache_enabled;
+ cp->avg_obj_size = config_vol->avg_obj_size;
config_vol->cachep = cp;
} else {
/* delete this volume from all the disks */
diff --git a/src/iocore/cache/P_CacheHosting.h
b/src/iocore/cache/P_CacheHosting.h
index 9e83e09db4..ef111cc899 100644
--- a/src/iocore/cache/P_CacheHosting.h
+++ b/src/iocore/cache/P_CacheHosting.h
@@ -296,6 +296,7 @@ struct ConfigVol {
bool in_percent;
bool ramcache_enabled;
int percent;
+ int avg_obj_size;
CacheVol *cachep;
LINK(ConfigVol, link);
};
diff --git a/src/iocore/cache/Stripe.cc b/src/iocore/cache/Stripe.cc
index 628249643e..90e33ebbef 100644
--- a/src/iocore/cache/Stripe.cc
+++ b/src/iocore/cache/Stripe.cc
@@ -84,7 +84,7 @@ struct StripeInitInfo {
// Stripe
//
-Stripe::Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip)
+Stripe::Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size)
: path{ats_strdup(disk->path)},
fd{disk->fd},
skip{ROUND_TO_STORE_BLOCK((dir_skip < START_POS ? START_POS : dir_skip))},
@@ -95,7 +95,7 @@ Stripe::Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip)
ink_assert(this->len < MAX_STRIPE_SIZE);
this->_init_hash_text(disk->path, blocks, dir_skip);
- this->_init_data(STORE_BLOCK_SIZE);
+ this->_init_data(STORE_BLOCK_SIZE, avg_obj_size);
this->_init_directory(this->dirlen(), this->headerlen(),
DIRECTORY_FOOTER_SIZE);
}
@@ -113,21 +113,25 @@ Stripe::_init_hash_text(char const *seed, off_t blocks,
off_t dir_skip)
}
void
-Stripe::_init_data(off_t store_block_size)
+Stripe::_init_data(off_t store_block_size, int avg_obj_size)
{
// iteratively calculate start + buckets; updates this->start
- this->_init_data_internal();
- this->_init_data_internal();
- this->_init_data_internal();
+ this->_init_data_internal(avg_obj_size);
+ this->_init_data_internal(avg_obj_size);
+ this->_init_data_internal(avg_obj_size);
this->data_blocks = (this->len - (this->start - this->skip)) /
store_block_size;
}
void
-Stripe::_init_data_internal()
+Stripe::_init_data_internal(int avg_obj_size)
{
+ if (avg_obj_size <= 0) {
+ avg_obj_size = cache_config_min_average_object_size;
+ }
+
// step1: calculate the number of entries.
- off_t total_entries = (this->len - (this->start - this->skip)) /
cache_config_min_average_object_size;
+ off_t total_entries = (this->len - (this->start - this->skip)) /
avg_obj_size;
// step2: calculate the number of buckets
off_t total_buckets = total_entries / DIR_DEPTH;
// step3: calculate the number of segments, no segment has more than 16384
buckets
diff --git a/src/iocore/cache/Stripe.h b/src/iocore/cache/Stripe.h
index ebf6e46dba..590d7883d9 100644
--- a/src/iocore/cache/Stripe.h
+++ b/src/iocore/cache/Stripe.h
@@ -51,6 +51,7 @@ struct CacheVol {
int scheme = 0;
off_t size = 0;
int num_vols = 0;
+ int avg_obj_size = -1; // Defer to the records.config if not
overriden
bool ramcache_enabled = true;
StripeSM **stripes = nullptr;
DiskStripe **disk_stripes = nullptr;
@@ -114,7 +115,7 @@ public:
*
* @see START_POS
*/
- Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip);
+ Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size = -1);
int dir_check();
@@ -176,8 +177,8 @@ protected:
private:
void _init_hash_text(char const *seed, off_t blocks, off_t dir_skip);
- void _init_data(off_t store_block_size);
- void _init_data_internal();
+ void _init_data(off_t store_block_size, int avg_obj_size = -1);
+ void _init_data_internal(int avg_obj_size = -1); // Defaults to
cache_config_min_average_object_size;
void _init_directory(std::size_t directory_size, int header_size, int
footer_size);
};
diff --git a/src/iocore/cache/StripeSM.cc b/src/iocore/cache/StripeSM.cc
index d5899dbe29..36a5d61229 100644
--- a/src/iocore/cache/StripeSM.cc
+++ b/src/iocore/cache/StripeSM.cc
@@ -112,8 +112,8 @@ struct StripeInitInfo {
// This is weird: the len passed to the constructor for _preserved_dirs is
// initialized in the superclasse's constructor. This is safe because the
// superclass should always be initialized first.
-StripeSM::StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip)
- : Continuation(new_ProxyMutex()), Stripe{disk, blocks, dir_skip},
_preserved_dirs{static_cast<int>(len)}
+StripeSM::StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int
avg_obj_size)
+ : Continuation(new_ProxyMutex()), Stripe{disk, blocks, dir_skip,
avg_obj_size}, _preserved_dirs{static_cast<int>(len)}
{
open_dir.mutex = this->mutex;
SET_HANDLER(&StripeSM::aggWrite);
diff --git a/src/iocore/cache/StripeSM.h b/src/iocore/cache/StripeSM.h
index 685942c49d..e24b89e497 100644
--- a/src/iocore/cache/StripeSM.h
+++ b/src/iocore/cache/StripeSM.h
@@ -168,7 +168,7 @@ public:
*
* @see START_POS
*/
- StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip);
+ StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size =
-1);
Queue<CacheVC, Continuation::Link_link> &get_pending_writers();
