Hello again,
Here's part b rebased, pgindented and with some minor additional tweaks (mostly function commands and the function renames I mentioned).
Patch looks ok to me, various tests where ok as well.
Still concerned about the unlocked stat accums.
See my arguments in other mail. I can add a lock if this is a blocker, but I think that it is actually better without, because of quantum: the measuring process should avoid affecting the measured data, and locking is not cheap.
I haven't tried to rebase the other ones yet, they need manual conflict fixes.
Find attached 14-c/d/e rebased patches.About e, for some obscure reason I failed in my initial attempt at inserting the misplaced options in their rightfull position in the option list. Sorry for the noise.
-- Fabien.
diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index 42d0667..ade1b53 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -1138,6 +1138,9 @@ number of transactions actually processed: 10000/10000 tps = 618.764555 (including connections establishing) tps = 622.977698 (excluding connections establishing) SQL script 1: <builtin: TPC-B (sort of)> + - 10000 transactions (100.0% of total, tps = 618.764555) + - latency average = 15.844 ms + - latency stddev = 2.715 ms - statement latencies in milliseconds: 0.004386 \set nbranches 1 * :scale 0.001343 \set ntellers 10 * :scale diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 305c319..5594d1c 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -164,6 +164,7 @@ bool use_log; /* log transaction latencies to a file */ bool use_quiet; /* quiet logging onto stderr */ int agg_interval; /* log aggregates instead of individual * transactions */ +bool per_script_stats = false; /* whether to collect stats per script */ int progress = 0; /* thread progress report every this seconds */ bool progress_timestamp = false; /* progress report with Unix time */ int nclients = 1; /* number of clients */ @@ -299,6 +300,7 @@ static struct { const char *name; Command **commands; + StatsData stats; } sql_script[MAX_SCRIPTS]; /* SQL script files */ static int num_scripts; /* number of scripts in sql_script[] */ static int num_commands = 0; /* total number of Command structs */ @@ -1308,7 +1310,7 @@ top: /* transaction finished: calculate latency and log the transaction */ if (commands[st->state + 1] == NULL) { - if (progress || throttle_delay || latency_limit || logfile) + if (progress || throttle_delay || latency_limit || per_script_stats || logfile) doTxStats(thread, st, &now, false, logfile, agg); else thread->stats.cnt++; @@ -1401,7 +1403,7 @@ top: } /* Record transaction start time under logging, progress or throttling */ - if ((logfile || progress || throttle_delay || latency_limit) && st->state == 0) + if ((logfile || progress || throttle_delay || latency_limit || per_script_stats) && st->state == 0) { INSTR_TIME_SET_CURRENT(st->txn_begin); @@ -1889,6 +1891,9 @@ doTxStats(TState *thread, CState *st, instr_time *now, if (use_log) doLog(thread, st, logfile, now, agg, skipped, latency, lag); + + if (per_script_stats) /* mutex? hmmm... these are only statistics */ + doStats(& sql_script[st->use_file].stats, skipped, latency, lag); } @@ -2678,6 +2683,7 @@ addScript(const char *name, Command **commands) sql_script[num_scripts].name = name; sql_script[num_scripts].commands = commands; + initStats(& sql_script[num_scripts].stats, 0.0); num_scripts++; } @@ -2761,22 +2767,40 @@ printResults(TState *threads, StatsData *total, instr_time total_time, printf("tps = %f (including connections establishing)\n", tps_include); printf("tps = %f (excluding connections establishing)\n", tps_exclude); - /* Report per-command latencies */ - if (is_latencies) + /* Report per-script stats */ + if (per_script_stats) { int i; for (i = 0; i < num_scripts; i++) { - Command **commands; + printf("SQL script %d: %s\n" + " - "INT64_FORMAT" transactions (%.1f%% of total, tps = %f)\n", + i+1, sql_script[i].name, + sql_script[i].stats.cnt, + 100.0 * sql_script[i].stats.cnt / total->cnt, + sql_script[i].stats.cnt / time_include); - printf("SQL script %d: %s\n", i + 1, sql_script[i].name); - printf(" - statement latencies in milliseconds:\n"); + if (latency_limit) + printf(" - number of transactions skipped: "INT64_FORMAT" (%.3f%%)\n", + sql_script[i].stats.skipped, + 100.0 * sql_script[i].stats.skipped / + (sql_script[i].stats.skipped + sql_script[i].stats.cnt)); - for (commands = sql_script[i].commands; *commands != NULL; commands++) - printf(" %11.3f %s\n", - 1000.0 * (*commands)->stats.sum / (*commands)->stats.count, - (*commands)->line); + printSimpleStats(" - latency", & sql_script[i].stats.latency); + + /* Report per-command latencies */ + if (is_latencies) + { + Command ** com; + + printf(" - per command latencies in ms:\n"); + + for (com = sql_script[i].commands; *com != NULL; com++) + printf(" %11.3f %s\n", + 1000.0 * (*com)->stats.sum / (*com)->stats.count, + (*com)->line); + } } } } @@ -2962,6 +2986,7 @@ main(int argc, char **argv) break; case 'r': benchmarking_option_set = true; + per_script_stats = true; is_latencies = true; break; case 's': @@ -3185,6 +3210,10 @@ main(int argc, char **argv) internal_script_used = true; } + /* show per script stats if several scripts are used */ + if (num_scripts > 1) + per_script_stats = true; + /* * Don't need more threads than there are clients. (This is not merely an * optimization; throttle_delay is calculated incorrectly below if some
diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index ade1b53..ca3e158 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -262,11 +262,13 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</> <variablelist> <varlistentry> - <term><option>-b</> <replaceable>scriptname</></term> - <term><option>--builtin</> <replaceable>scriptname</></term> + <term><option>-b</> <replaceable>scriptname[@weight]</></term> + <term><option>--builtin</> <replaceable>scriptname[@weight]</></term> <listitem> <para> Add the specified builtin script to the list of executed scripts. + An optional integer weight after <literal>@</> allows to adjust the + probability of drawing the test. Available builtin scripts are: <literal>tpcb-like</>, <literal>simple-update</> and <literal>select-only</>. With special name <literal>list</>, show the list of builtin scripts @@ -321,12 +323,14 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</> </varlistentry> <varlistentry> - <term><option>-f</> <replaceable>filename</></term> - <term><option>--file=</><replaceable>filename</></term> + <term><option>-f</> <replaceable>filename[@weight]</></term> + <term><option>--file=</><replaceable>filename[@weight]</></term> <listitem> <para> Add a transaction script read from <replaceable>filename</> to the list of executed scripts. + An optional integer weight after <literal>@</> allows to adjust the + probability of drawing the test. See below for details. </para> </listitem> @@ -689,6 +693,9 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</> Pgbench executes test scripts chosen randomly from a specified list. They include built-in scripts with <option>-b</> and user-provided custom scripts with <option>-f</>. + Each script may be given a relative weight specified after a + <literal>@</> so as to change its drawing probability. + The default weight is <literal>1</>. </para> <para> @@ -1137,7 +1144,7 @@ number of transactions per client: 1000 number of transactions actually processed: 10000/10000 tps = 618.764555 (including connections establishing) tps = 622.977698 (excluding connections establishing) -SQL script 1: <builtin: TPC-B (sort of)> +SQL script 1, weight 1: <builtin: TPC-B (sort of)> - 10000 transactions (100.0% of total, tps = 618.764555) - latency average = 15.844 ms - latency stddev = 2.715 ms diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 5594d1c..7aa6db7 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -179,6 +179,8 @@ char *login = NULL; char *dbName; const char *progname; +#define WSEP '@' /* weight separator */ + volatile bool timer_exceeded = false; /* flag from signal handler */ /* variable definitions */ @@ -299,11 +301,14 @@ typedef struct static struct { const char *name; + int weight; Command **commands; StatsData stats; } sql_script[MAX_SCRIPTS]; /* SQL script files */ static int num_scripts; /* number of scripts in sql_script[] */ static int num_commands = 0; /* total number of Command structs */ +static int total_weight = 0; + static int debug = 0; /* debug flag */ /* Define builtin test scripts */ @@ -385,9 +390,9 @@ usage(void) " --tablespace=TABLESPACE create tables in the specified tablespace\n" " --unlogged-tables create tables as unlogged tables\n" "\nOptions to select what to run:\n" - " -b, --builtin=NAME add buitin script (use \"-b list\" to display\n" - " available scripts)\n" - " -f, --file=FILENAME add transaction script from FILENAME\n" + " -b, --builtin=NAME[@W] add weighted buitin script (use \"-b list\"\n" + " to display available scripts)\n" + " -f, --file=FILENAME[@W] add weighted transaction script from FILENAME\n" " -N, --skip-some-updates skip updates of pgbench_tellers and pgbench_branches\n" " (same as \"-b simple-update\")\n" " -S, --select-only perform SELECT-only transactions\n" @@ -1183,10 +1188,17 @@ clientDone(CState *st, bool ok) static int chooseScript(TState *thread) { + int i = 0, w = 0, wc; + if (num_scripts == 1) return 0; - return getrand(thread, 0, num_scripts - 1); + wc = (int) getrand(thread, 0, total_weight - 1); + do { + w += sql_script[i++].weight; + } while (w <= wc); + + return i - 1; } /* return false iff client should be disconnected */ @@ -2666,8 +2678,41 @@ findBuiltin(const char *name, char **desc) exit(1); } +/* Possiby truncate option and return weight */ +static int +getWeight(char *option) +{ + char *sep; + int weight; + + if ((sep = strrchr(option, WSEP))) + { + char *s; + *sep++ = '\0'; + + /* check that the weight is a positive integer */ + s = sep; + while ('0' <= *s && *s <= '9') + s++; + if (*s != '\0' || s == sep) + { + /* empty or any other char */ + fprintf(stderr, + "weight for script \"%s\" must be an integer, got \"%s\"\n", + option, sep); + exit(1); + } + + weight = atoi(sep); + } + else + weight = 1; + + return weight; +} + static void -addScript(const char *name, Command **commands) +addScript(const char *name, Command **commands, int weight) { if (commands == NULL) { @@ -2682,6 +2727,7 @@ addScript(const char *name, Command **commands) } sql_script[num_scripts].name = name; + sql_script[num_scripts].weight = weight; sql_script[num_scripts].commands = commands; initStats(& sql_script[num_scripts].stats, 0.0); num_scripts++; @@ -2767,16 +2813,16 @@ printResults(TState *threads, StatsData *total, instr_time total_time, printf("tps = %f (including connections establishing)\n", tps_include); printf("tps = %f (excluding connections establishing)\n", tps_exclude); - /* Report per-script stats */ + /* Report per-file data */ if (per_script_stats) { int i; for (i = 0; i < num_scripts; i++) { - printf("SQL script %d: %s\n" + printf("SQL script %d, weight %d: %s\n" " - "INT64_FORMAT" transactions (%.1f%% of total, tps = %f)\n", - i+1, sql_script[i].name, + i+1, sql_script[i].weight, sql_script[i].name, sql_script[i].stats.cnt, 100.0 * sql_script[i].stats.cnt / total->cnt, sql_script[i].stats.cnt / time_include); @@ -2867,6 +2913,7 @@ main(int argc, char **argv) instr_time conn_total_time; int64 latency_late = 0; StatsData stats; + int weight; char *desc; int i; @@ -3046,27 +3093,32 @@ main(int argc, char **argv) exit(0); } + weight = getWeight(optarg); addScript(desc, - process_builtin(findBuiltin(optarg, &desc), desc)); + process_builtin(findBuiltin(optarg, &desc), desc), + weight); benchmarking_option_set = true; internal_script_used = true; break; case 'S': addScript(desc, process_builtin(findBuiltin("select-only", &desc), - desc)); + desc), + 1); benchmarking_option_set = true; internal_script_used = true; break; case 'N': addScript(desc, process_builtin(findBuiltin("simple-update", &desc), - desc)); + desc), + 1); benchmarking_option_set = true; internal_script_used = true; break; case 'f': - addScript(optarg, process_file(optarg)); + weight = getWeight(optarg); + addScript(optarg, process_file(optarg), weight); benchmarking_option_set = true; break; case 'D': @@ -3205,11 +3257,16 @@ main(int argc, char **argv) if (num_scripts == 0 && !is_init_mode) { addScript(desc, - process_builtin(findBuiltin("tpcb-like", &desc), desc)); + process_builtin(findBuiltin("tpcb-like", &desc), desc), + 1); benchmarking_option_set = true; internal_script_used = true; } + /* compute total_weight */ + for (i = 0; i < num_scripts; i++) + total_weight += sql_script[i].weight; + /* show per script stats if several scripts are used */ if (num_scripts > 1) per_script_stats = true;
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 7aa6db7..a4f9abf 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -1280,7 +1280,7 @@ top: if (INSTR_TIME_GET_MICROSEC(now) < st->txn_scheduled) return true; /* Still sleeping, nothing to do here */ /* Else done sleeping, go ahead with next command */ - st->sleeping = 0; + st->sleeping = false; st->throttling = false; } @@ -2867,15 +2867,15 @@ main(int argc, char **argv) {"host", required_argument, NULL, 'h'}, {"initialize", no_argument, NULL, 'i'}, {"jobs", required_argument, NULL, 'j'}, - {"log", no_argument, NULL, 'l'}, {"latency-limit", required_argument, NULL, 'L'}, + {"log", no_argument, NULL, 'l'}, {"no-vacuum", no_argument, NULL, 'n'}, {"port", required_argument, NULL, 'p'}, {"progress", required_argument, NULL, 'P'}, {"protocol", required_argument, NULL, 'M'}, {"quiet", no_argument, NULL, 'q'}, - {"report-latencies", no_argument, NULL, 'r'}, {"rate", required_argument, NULL, 'R'}, + {"report-latencies", no_argument, NULL, 'r'}, {"scale", required_argument, NULL, 's'}, {"select-only", no_argument, NULL, 'S'}, {"skip-some-updates", no_argument, NULL, 'N'},
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers