This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 675b9f4c29 Adds a fragment-size= option to volume.config (#11865)
675b9f4c29 is described below
commit 675b9f4c2933725a9957c771e202bb9d0c5eb71c
Author: Leif Hedstrom <[email protected]>
AuthorDate: Fri Nov 15 09:15:53 2024 -0700
Adds a fragment-size= option to volume.config (#11865)
* Adds a fragment-size= option to volume.config
* Fix docstring
---
doc/admin-guide/files/volume.config.en.rst | 13 ++++++++++++-
src/iocore/cache/Cache.cc | 2 +-
src/iocore/cache/CacheHosting.cc | 9 +++++++++
src/iocore/cache/CacheProcessor.cc | 1 +
src/iocore/cache/CacheWrite.cc | 13 ++++++++-----
src/iocore/cache/P_CacheHosting.h | 1 +
src/iocore/cache/Stripe.cc | 3 ++-
src/iocore/cache/Stripe.h | 4 +++-
src/iocore/cache/StripeSM.cc | 6 ++++--
src/iocore/cache/StripeSM.h | 6 +++++-
10 files changed, 46 insertions(+), 12 deletions(-)
diff --git a/doc/admin-guide/files/volume.config.en.rst
b/doc/admin-guide/files/volume.config.en.rst
index 91fc2e648a..70ec2725af 100644
--- a/doc/admin-guide/files/volume.config.en.rst
+++ b/doc/admin-guide/files/volume.config.en.rst
@@ -76,6 +76,17 @@ line. This overrides the global
:ts:cv:`proxy.config.cache.min_average_object_si
configuration for this volume. This is useful if you have a volume that is
dedicated
for say very small objects, and you need a lot of directory entries to store
them.
+Optional fragment size setting
+------------------------------
+
+You can also add an option ``fragment_size=<size>`` to the volume configuration
+line. This overrides the global
:ts:cv:`proxy.config.cache.target_fragment_size`
+configuration for this volume. This allows for a smaller, or larger, fragment
size
+for a particular volume. This may be useful together with ``avg_obj_size`` as
well,
+since a larger fragment size could reduce the number of directory entries
needed
+for a large object.
+
+Note that this setting has a maximmum value of 4MB.
Exclusive spans and volume sizes
================================
@@ -116,4 +127,4 @@ ramcache has been disabled.::
volume=2 scheme=http size=20%
volume=3 scheme=http size=20%
volume=4 scheme=http size=20% avg_obj_size=4096
- volume=5 scheme=http size=20% ramcache=false
+ volume=5 scheme=http size=20% ramcache=false fragment_size=524288
diff --git a/src/iocore/cache/Cache.cc b/src/iocore/cache/Cache.cc
index d99a53f6da..1dc544194b 100644
--- a/src/iocore/cache/Cache.cc
+++ b/src/iocore/cache/Cache.cc
@@ -273,7 +273,7 @@ Cache::open(bool clear, bool /* fix ATS_UNUSED */)
for (; q; q = q->link.next) {
blocks = q->b->len;
CacheDisk *d = cp->disk_stripes[i]->disk;
- cp->stripes[vol_no] = new StripeSM(d, blocks,
q->b->offset, cp->avg_obj_size);
+ cp->stripes[vol_no] = new StripeSM(d, blocks,
q->b->offset, cp->avg_obj_size, cp->fragment_size);
cp->stripes[vol_no]->cache = this;
cp->stripes[vol_no]->cache_vol = cp;
diff --git a/src/iocore/cache/CacheHosting.cc b/src/iocore/cache/CacheHosting.cc
index e984d41f0b..7e85104797 100644
--- a/src/iocore/cache/CacheHosting.cc
+++ b/src/iocore/cache/CacheHosting.cc
@@ -655,6 +655,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
int in_percent = 0;
bool ramcache_enabled = true;
int avg_obj_size = -1; // Defaults
+ int fragment_size = -1;
while (true) {
// skip all blank spaces at beginning of line
@@ -746,6 +747,13 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
tmp += 13;
avg_obj_size = atoi(tmp);
+ while (ParseRules::is_digit(*tmp)) {
+ tmp++;
+ }
+ } else if (strcasecmp(tmp, "fragment_size") == 0) { // match
fragment_size
+ tmp += 14;
+ fragment_size = atoi(tmp);
+
while (ParseRules::is_digit(*tmp)) {
tmp++;
}
@@ -786,6 +794,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path,
char *file_buf)
configp->scheme = scheme;
configp->size = size;
configp->avg_obj_size = avg_obj_size;
+ configp->fragment_size = fragment_size;
configp->cachep = nullptr;
configp->ramcache_enabled = ramcache_enabled;
cp_queue.enqueue(configp);
diff --git a/src/iocore/cache/CacheProcessor.cc
b/src/iocore/cache/CacheProcessor.cc
index 402fc619c5..a1ce9a0eba 100644
--- a/src/iocore/cache/CacheProcessor.cc
+++ b/src/iocore/cache/CacheProcessor.cc
@@ -1178,6 +1178,7 @@ cplist_update()
if (cp->scheme == config_vol->scheme) {
cp->ramcache_enabled = config_vol->ramcache_enabled;
cp->avg_obj_size = config_vol->avg_obj_size;
+ cp->fragment_size = config_vol->fragment_size;
config_vol->cachep = cp;
} else {
/* delete this volume from all the disks */
diff --git a/src/iocore/cache/CacheWrite.cc b/src/iocore/cache/CacheWrite.cc
index e31015032d..613a4085a8 100644
--- a/src/iocore/cache/CacheWrite.cc
+++ b/src/iocore/cache/CacheWrite.cc
@@ -525,9 +525,10 @@ CacheVC::openWriteWriteDone(int event, Event *e)
}
static inline int
-target_fragment_size()
+target_fragment_size(int target_frag_size)
{
- uint64_t value = cache_config_target_fragment_size - sizeof(Doc);
+ uint64_t value = (target_frag_size > 0 ? target_frag_size :
cache_config_target_fragment_size) - sizeof(Doc);
+
ink_release_assert(value <= MAX_FRAG_SIZE);
return value;
}
@@ -561,6 +562,8 @@ Lagain:
int64_t total_avail = vio.get_reader()->read_avail();
int64_t avail = total_avail;
int64_t towrite = avail + length;
+ int frag_size = target_fragment_size(stripe->frag_size);
+
if (towrite > ntodo) {
avail -= (towrite - ntodo);
towrite = ntodo;
@@ -579,12 +582,12 @@ Lagain:
total_len += avail;
}
length = static_cast<uint64_t>(towrite);
- if (length > target_fragment_size() && (length < target_fragment_size() +
target_fragment_size() / 4)) {
- write_len = target_fragment_size();
+ if (length > frag_size && (length < frag_size + frag_size / 4)) {
+ write_len = frag_size;
} else {
write_len = length;
}
- bool not_writing = towrite != ntodo && towrite < target_fragment_size();
+ bool not_writing = towrite != ntodo && towrite < frag_size;
if (!called_user) {
if (not_writing) {
called_user = 1;
diff --git a/src/iocore/cache/P_CacheHosting.h
b/src/iocore/cache/P_CacheHosting.h
index ef111cc899..2f98295caa 100644
--- a/src/iocore/cache/P_CacheHosting.h
+++ b/src/iocore/cache/P_CacheHosting.h
@@ -297,6 +297,7 @@ struct ConfigVol {
bool ramcache_enabled;
int percent;
int avg_obj_size;
+ int fragment_size;
CacheVol *cachep;
LINK(ConfigVol, link);
};
diff --git a/src/iocore/cache/Stripe.cc b/src/iocore/cache/Stripe.cc
index 90e33ebbef..7f9aaf9338 100644
--- a/src/iocore/cache/Stripe.cc
+++ b/src/iocore/cache/Stripe.cc
@@ -84,9 +84,10 @@ struct StripeInitInfo {
// Stripe
//
-Stripe::Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size)
+Stripe::Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int
avg_obj_size, int fragment_size)
: path{ats_strdup(disk->path)},
fd{disk->fd},
+ frag_size{fragment_size},
skip{ROUND_TO_STORE_BLOCK((dir_skip < START_POS ? START_POS : dir_skip))},
start{skip},
len{blocks * STORE_BLOCK_SIZE},
diff --git a/src/iocore/cache/Stripe.h b/src/iocore/cache/Stripe.h
index 590d7883d9..de31857b5b 100644
--- a/src/iocore/cache/Stripe.h
+++ b/src/iocore/cache/Stripe.h
@@ -52,6 +52,7 @@ struct CacheVol {
off_t size = 0;
int num_vols = 0;
int avg_obj_size = -1; // Defer to the records.config if not
overriden
+ int fragment_size = -1; // Defer to the records.config if not
overriden
bool ramcache_enabled = true;
StripeSM **stripes = nullptr;
DiskStripe **disk_stripes = nullptr;
@@ -86,6 +87,7 @@ public:
ats_scoped_str hash_text;
char *path = nullptr;
int fd{-1};
+ int frag_size{-1};
char *raw_dir{nullptr};
Dir *dir{};
@@ -115,7 +117,7 @@ public:
*
* @see START_POS
*/
- Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size = -1);
+ Stripe(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size = -1,
int fragment_size = -1);
int dir_check();
diff --git a/src/iocore/cache/StripeSM.cc b/src/iocore/cache/StripeSM.cc
index 36a5d61229..c7fe33ef1f 100644
--- a/src/iocore/cache/StripeSM.cc
+++ b/src/iocore/cache/StripeSM.cc
@@ -112,8 +112,10 @@ struct StripeInitInfo {
// This is weird: the len passed to the constructor for _preserved_dirs is
// initialized in the superclasse's constructor. This is safe because the
// superclass should always be initialized first.
-StripeSM::StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int
avg_obj_size)
- : Continuation(new_ProxyMutex()), Stripe{disk, blocks, dir_skip,
avg_obj_size}, _preserved_dirs{static_cast<int>(len)}
+StripeSM::StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int
avg_obj_size, int fragment_size)
+ : Continuation(new_ProxyMutex()),
+ Stripe{disk, blocks, dir_skip, avg_obj_size, fragment_size},
+ _preserved_dirs{static_cast<int>(len)}
{
open_dir.mutex = this->mutex;
SET_HANDLER(&StripeSM::aggWrite);
diff --git a/src/iocore/cache/StripeSM.h b/src/iocore/cache/StripeSM.h
index a1d1effcb4..5a34adb724 100644
--- a/src/iocore/cache/StripeSM.h
+++ b/src/iocore/cache/StripeSM.h
@@ -165,10 +165,14 @@ public:
* @param blocks: Number of blocks. Must be at least 10.
* @param dir_skip: Offset into the disk at which to start the stripe.
* If this value is less than START_POS, START_POS will be used instead.
+ * @param avg_obj_size: Optional average object size. If not provided, use
default
+ * from proxy.config.cache.min_average_object_size.
+ * @param fragment_size: Optional fragment size. If not provided, use default
+ * from proxy.config.cache.target_fragment_size.
*
* @see START_POS
*/
- StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size =
-1);
+ StripeSM(CacheDisk *disk, off_t blocks, off_t dir_skip, int avg_obj_size =
-1, int fragment_size = -1);
Queue<CacheVC, Continuation::Link_link> &get_pending_writers();
