Edenhill has uploaded a new change for review.
https://gerrit.wikimedia.org/r/90028
Change subject: Make scratch buffer size configurable (issue #2)
......................................................................
Make scratch buffer size configurable (issue #2)
Controlled through the "log.line.scratch.size" property (default 4096)
Change-Id: I4a3f37d5b40bee4c33c7ba1bb4fa7a1c35bfb613
---
M config.c
M varnishkafka.c
M varnishkafka.conf.example
M varnishkafka.h
4 files changed, 18 insertions(+), 8 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/operations/software/varnish/varnishkafka
refs/changes/28/90028/1
diff --git a/config.c b/config.c
index ef90b69..98f15de 100644
--- a/config.c
+++ b/config.c
@@ -187,6 +187,8 @@
conf.loglines_hsize = atoi(val);
else if (!strcmp(name, "log.hash.max"))
conf.loglines_hmax = atoi(val);
+ else if (!strcmp(name, "log.line.scratch.size"))
+ conf.scratch_size = atoi(val);
else if (!strncmp(name, "varnish.arg.", strlen("varnish.arg."))) {
const char *t = name + strlen("varnish.arg.");
int r = 0;
diff --git a/varnishkafka.c b/varnishkafka.c
index 37109f4..c5285a1 100644
--- a/varnishkafka.c
+++ b/varnishkafka.c
@@ -274,7 +274,7 @@
* Returns true if 'ptr' is within 'lp's scratch pad, else false.
*/
static inline int is_scratch_ptr (const struct logline *lp, const char *ptr) {
- return (lp->scratch <= ptr && ptr < lp->scratch + sizeof(lp->scratch));
+ return (lp->scratch <= ptr && ptr < lp->scratch + conf.scratch_size);
}
/**
@@ -294,11 +294,11 @@
int len) {
char *ptr;
- if (lp->sof + len > sizeof(lp->scratch)) {
+ if (lp->sof + len > conf.scratch_size) {
vk_log("WARNING", LOG_WARNING,
"scratch pad is too small (%zd bytes), "
- "need %i bytes or more",
- sizeof(lp->scratch), lp->sof + len);
+ "need %i bytes or more: increase log.line.scratch.size",
+ conf.scratch_size, lp->sof + len);
return NULL;
}
@@ -1434,10 +1434,10 @@
}
/* Allocate and set up new logline */
- lp = calloc(1, sizeof(*lp) +
+ lp = calloc(1, sizeof(*lp) + conf.scratch_size +
(conf.total_fmt_cnt * sizeof(*lp->match[0])));
lp->id = id;
- ptr = (char *)(lp+1);
+ ptr = (char *)(lp+1) + conf.scratch_size;
for (i = 0 ; i < conf.fconf_cnt ; i++) {
lp->match[i] = (struct match *)ptr;
ptr += conf.fconf[i].fmt_cnt * sizeof(*lp->match[i]);
@@ -1653,6 +1653,7 @@
conf.datacopy = 1;
conf.loglines_hsize = 5000;
conf.loglines_hmax = 5;
+ conf.scratch_size = 4096;
conf.rk_conf = rd_kafka_conf_new();
rd_kafka_conf_set(conf.rk_conf, "client.id", "varnishkafka", NULL, 0);
rd_kafka_conf_set_error_cb(conf.rk_conf, kafka_error_cb);
diff --git a/varnishkafka.conf.example b/varnishkafka.conf.example
index 04fbb79..2d14972 100644
--- a/varnishkafka.conf.example
+++ b/varnishkafka.conf.example
@@ -120,7 +120,6 @@
# Logline cache hash tuning
# 'log.hash.size * log.hash.max' dictates the maximum number of cached logline
# entries in memory.
-#
# Number of hash buckets (keyed by log id).
# Higher number yields more performance at the expense of memory.
@@ -134,6 +133,12 @@
# Defaults to 5
#log.hash.max = 5
+# Size of per logline scratch buffer.
+# The scratch buffer is used as a temporary storage space while
+# collecting tags for the log line.
+# If the scratch size is too small the logline tag match will be incomplete.
+# Defaults to 4096 bytes.
+#log.line.scratch.size = 4096
# Start for sequence number (%n)
diff --git a/varnishkafka.h b/varnishkafka.h
index 0b71653..685dffd 100644
--- a/varnishkafka.h
+++ b/varnishkafka.h
@@ -88,7 +88,8 @@
/* Scratch pad */
int sof;
- char scratch[2048]; /* Must be at end of struct */
+ char scratch[0]; /* Must be at end of struct.
+ * Allocated to conf.scratch_size bytes */
};
@@ -165,6 +166,7 @@
uint64_t sequence_number;
+ size_t scratch_size; /* Size of scratch buffer */
int datacopy;
fmt_enc_t fmt_enc;
int total_fmt_cnt;
--
To view, visit https://gerrit.wikimedia.org/r/90028
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4a3f37d5b40bee4c33c7ba1bb4fa7a1c35bfb613
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/varnish/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Edenhill <mag...@edenhill.se>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
