Currently every histogram of a region accounts every I/O. To later support histograms that only account a particular kind of I/O, classify each once per accounting call into a bitmask of classes and give each histogram a class_mask of the classes it accounts.
The per-class dispatch is precomputed at region creation time into a small lookup table indexed by the class bitmask, so the hot path selects the histograms to update with a single table load instead of per-histogram filter comparisons. dm_stats_bin_duration() then walks only the set bits of the resulting histogram mask. This commit does not introduce any histogram types yet, but let's imagine we have 'write', 'all' and 'read' histograms. Then the table dm_stat->hist_mask will look like that: [0b00] = 0b000 [0b01] (read) = 0b110 [0b10] (write) = 0b011 [0b11] (read & write) = 0b111 For WRITE request (class 0b10) during the binning we will iterate over __ffs(0b011) and add the duration value into hist#0 (write) and hist#1 (all). https://virtuozzo.atlassian.net/browse/VSTOR-103846 Signed-off-by: Andrey Zhadchenko <[email protected]> --- drivers/md/dm-stats.c | 74 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index 21109ccfa10a9..51b3c736b64c5 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -39,9 +39,23 @@ struct dm_stat_shared { struct dm_stat_percpu tmp; }; +/* + * Histograms account I/O classes independently. An I/O is classified once + * per dm_stats_account_io() and may belong to several classes at once. + */ +enum { + DM_STAT_CLASS_READ, + DM_STAT_CLASS_WRITE, + DM_STAT_NR_CLASSES, +}; + +#define DM_STAT_CLASSES_DATA ((1 << DM_STAT_CLASS_READ) | \ + (1 << DM_STAT_CLASS_WRITE)) + #define DM_STAT_MAX_HISTOGRAMS 1 struct dm_stat_histogram { + unsigned int class_mask; /* classes this histogram accounts */ unsigned int n_entries; unsigned int offset; unsigned long long *boundaries; @@ -57,6 +71,8 @@ struct dm_stat { sector_t step; unsigned int n_histograms; unsigned int histogram_buckets; + /* class bitmask -> bitmask of histograms accounting these classes */ + u8 hist_mask[1 << DM_STAT_NR_CLASSES]; struct dm_stat_histogram histograms[DM_STAT_MAX_HISTOGRAMS]; const char *program_id; const char *aux_data; @@ -351,6 +367,12 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, s->n_histograms = n_histograms; s->histogram_buckets = histogram_buckets; for (hn = 0; hn < n_histograms; hn++) { + unsigned int cm; + + s->histograms[hn].class_mask = histograms[hn].class_mask; + for (cm = 1; cm < ARRAY_SIZE(s->hist_mask); cm++) + if (histograms[hn].class_mask & cm) + s->hist_mask[cm] |= 1 << hn; s->histograms[hn].n_entries = histograms[hn].n_entries; s->histograms[hn].offset = (hn ? s->histograms[hn - 1].offset + s->histograms[hn - 1].n_entries + 1 : 0); @@ -596,17 +618,27 @@ static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, shared->stamp = now; } +static unsigned int dm_stat_io_classes(blk_opf_t bi_opf, unsigned int bi_sectors) +{ + unsigned int classes = 0; + + if (bi_sectors) + classes |= 1 << (op_is_write(bi_opf) ? DM_STAT_CLASS_WRITE : + DM_STAT_CLASS_READ); + + return classes; +} static void dm_stats_bin_duration(struct dm_stat *s, struct dm_stat_percpu *p, + unsigned int hmask, unsigned long long duration) { - unsigned int hn; - - - for (hn = 0; hn < s->n_histograms; hn++) { - struct dm_stat_histogram *h = &s->histograms[hn]; + do { + struct dm_stat_histogram *h = &s->histograms[__ffs(hmask)]; unsigned int lo = 0, hi = h->n_entries + 1; + hmask &= hmask - 1; + while (lo + 1 < hi) { unsigned int mid = (lo + hi) / 2; @@ -616,12 +648,13 @@ static void dm_stats_bin_duration(struct dm_stat *s, struct dm_stat_percpu *p, lo = mid; } p->histogram[h->offset + lo]++; - } + } while (hmask); } static void dm_stat_for_entry(struct dm_stat *s, size_t entry, - blk_opf_t bi_opf, sector_t len, + blk_opf_t bi_opf, unsigned int classes, + sector_t len, struct dm_stats_aux *stats_aux, bool end, unsigned long duration_jiffies) { @@ -658,6 +691,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, atomic_inc(&shared->in_flight[idx]); } else { unsigned long long duration; + unsigned int hmask = s->hist_mask[classes]; dm_stat_round(s, shared, p); atomic_dec(&shared->in_flight[idx]); @@ -671,12 +705,12 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, p->ticks[idx] += stats_aux->duration_ns; duration = stats_aux->duration_ns; } - if (s->n_histograms) { + if (hmask) { if ((s->stat_flags & STAT_HIST_TOTAL_LATENCY) && stats_aux->histogram_duration_ns) duration = stats_aux->histogram_duration_ns; - dm_stats_bin_duration(s, p, duration); + dm_stats_bin_duration(s, p, hmask, duration); } } @@ -688,6 +722,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, } static void __dm_stat_bio(struct dm_stat *s, blk_opf_t bi_opf, + unsigned int classes, sector_t bi_sector, sector_t end_sector, bool end, unsigned long duration_jiffies, struct dm_stats_aux *stats_aux) @@ -717,7 +752,7 @@ static void __dm_stat_bio(struct dm_stat *s, blk_opf_t bi_opf, fragment_len = todo; if (fragment_len > s->step - offset) fragment_len = s->step - offset; - dm_stat_for_entry(s, entry, bi_opf, fragment_len, + dm_stat_for_entry(s, entry, bi_opf, classes, fragment_len, stats_aux, end, duration_jiffies); todo -= fragment_len; entry++; @@ -736,8 +771,9 @@ void dm_stats_account_io(struct dm_stats *stats, blk_opf_t bi_opf, bool got_precise_time; unsigned long duration_jiffies = 0; unsigned long bi_rw = op_is_write(bi_opf); + unsigned int classes = dm_stat_io_classes(bi_opf, bi_sectors); - if (unlikely(!bi_sectors)) + if (unlikely(!classes)) return; end_sector = bi_sector + bi_sectors; @@ -768,7 +804,8 @@ void dm_stats_account_io(struct dm_stats *stats, blk_opf_t bi_opf, stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns; got_precise_time = true; } - __dm_stat_bio(s, bi_opf, bi_sector, end_sector, end, duration_jiffies, stats_aux); + __dm_stat_bio(s, bi_opf, classes, bi_sector, end_sector, end, + duration_jiffies, stats_aux); } rcu_read_unlock(); @@ -1059,14 +1096,20 @@ static int parse_histogram(const char *h, unsigned int *n_histogram_entries, } static int add_histogram(struct dm_stat_histogram *histograms, - unsigned int *n_histograms, const char *h) + unsigned int *n_histograms, unsigned int class_mask, + const char *h) { - unsigned int idx; + unsigned int i, idx; + + for (i = 0; i < *n_histograms; i++) + if (histograms[i].class_mask == class_mask) + return -EINVAL; if (*n_histograms >= DM_STAT_MAX_HISTOGRAMS) return -EINVAL; idx = (*n_histograms)++; + histograms[idx].class_mask = class_mask; histograms[idx].boundaries = NULL; return parse_histogram(h, &histograms[idx].n_entries, @@ -1142,7 +1185,8 @@ static int message_stats_create(struct mapped_device *md, else if (!strcasecmp(a, "hist_total_latency")) stat_flags |= STAT_HIST_TOTAL_LATENCY; else if (!strncasecmp(a, "histogram:", 10)) { - r = add_histogram(histograms, &n_histograms, a + 10); + r = add_histogram(histograms, &n_histograms, + DM_STAT_CLASSES_DATA, a + 10); if (r) goto ret; } else -- 2.43.5 _______________________________________________ Devel mailing list [email protected] https://lists.openvz.org/mailman/listinfo/devel
