Module Name: src Committed By: joerg Date: Tue Sep 25 20:53:47 UTC 2012
Modified Files: src/tests/usr.bin/nbperf: h_nbperf.sh hash_driver.c t_nbperf.sh src/usr.bin/nbperf: nbperf-bdz.c nbperf.1 Log Message: Simplify the BDZ compression function, making it smaller at the same time. Fixes a bug where non-minimal hash functions could be created. Add regression tests for BDZ, including the map output functionality. To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 src/tests/usr.bin/nbperf/h_nbperf.sh \ src/tests/usr.bin/nbperf/hash_driver.c \ src/tests/usr.bin/nbperf/t_nbperf.sh cvs rdiff -u -r1.4 -r1.5 src/usr.bin/nbperf/nbperf-bdz.c \ src/usr.bin/nbperf/nbperf.1 Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/tests/usr.bin/nbperf/h_nbperf.sh diff -u src/tests/usr.bin/nbperf/h_nbperf.sh:1.1 src/tests/usr.bin/nbperf/h_nbperf.sh:1.2 --- src/tests/usr.bin/nbperf/h_nbperf.sh:1.1 Sun Jul 22 20:38:20 2012 +++ src/tests/usr.bin/nbperf/h_nbperf.sh Tue Sep 25 20:53:46 2012 @@ -1,5 +1,5 @@ #!/bin/sh -# $NetBSD: h_nbperf.sh,v 1.1 2012/07/22 20:38:20 joerg Exp $ +# $NetBSD: h_nbperf.sh,v 1.2 2012/09/25 20:53:46 joerg Exp $ # # Copyright (c) 2012 The NetBSD Foundation, Inc. # All rights reserved. @@ -27,6 +27,6 @@ # set -e -head -n $4 $1 | nbperf -a $2 > hash.c 2> /dev/null +head -n $4 $1 | nbperf -m hash.map -o hash.c -a $2 2> /dev/null cc -o testprog -I. $5 head -n $4 $1 | ./testprog | $3 Index: src/tests/usr.bin/nbperf/hash_driver.c diff -u src/tests/usr.bin/nbperf/hash_driver.c:1.1 src/tests/usr.bin/nbperf/hash_driver.c:1.2 --- src/tests/usr.bin/nbperf/hash_driver.c:1.1 Sun Jul 22 20:38:20 2012 +++ src/tests/usr.bin/nbperf/hash_driver.c Tue Sep 25 20:53:46 2012 @@ -1,4 +1,4 @@ -/* $NetBSD: hash_driver.c,v 1.1 2012/07/22 20:38:20 joerg Exp $ */ +/* $NetBSD: hash_driver.c,v 1.2 2012/09/25 20:53:46 joerg Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. * All rights reserved. @@ -46,7 +46,7 @@ main(void) while ((len = getline(&line, &len, stdin)) > 0) { if (len && line[len - 1] == '\n') --len; - printf("%" PRId32 "\n", 1 + hash(line, len)); + printf("%" PRId32 "\n", hash(line, len)); } free(line); return 0; Index: src/tests/usr.bin/nbperf/t_nbperf.sh diff -u src/tests/usr.bin/nbperf/t_nbperf.sh:1.1 src/tests/usr.bin/nbperf/t_nbperf.sh:1.2 --- src/tests/usr.bin/nbperf/t_nbperf.sh:1.1 Sun Jul 22 20:38:20 2012 +++ src/tests/usr.bin/nbperf/t_nbperf.sh Tue Sep 25 20:53:46 2012 @@ -1,4 +1,4 @@ -# $NetBSD: t_nbperf.sh,v 1.1 2012/07/22 20:38:20 joerg Exp $ +# $NetBSD: t_nbperf.sh,v 1.2 2012/09/25 20:53:46 joerg Exp $ # # Copyright (c) 2012 The NetBSD Foundation, Inc. # All rights reserved. @@ -27,7 +27,7 @@ cleanup() { - rm -f reference.txt hash.c testprog + rm -f reference.txt hash.c hash.map testprog } atf_test_case chm @@ -38,10 +38,13 @@ chm_head() chm_body() { for n in 4 32 128 1024 65536; do - seq $n > reference.txt + seq 0 $(($n - 1)) > reference.txt atf_check -o file:reference.txt \ $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 chm cat \ $n $(atf_get_srcdir)/hash_driver.c + atf_check -o file:hash.map \ + $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 chm cat \ + $n $(atf_get_srcdir)/hash_driver.c done } chm_clean() @@ -57,10 +60,13 @@ chm3_head() chm3_body() { for n in 4 32 128 1024 65536; do - seq $n > reference.txt + seq 0 $(($n - 1)) > reference.txt atf_check -o file:reference.txt \ $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 chm3 cat \ $n $(atf_get_srcdir)/hash_driver.c + atf_check -o file:hash.map \ + $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 chm3 cat \ + $n $(atf_get_srcdir)/hash_driver.c done } chm3_clean() @@ -68,8 +74,31 @@ chm3_clean() cleanup } +atf_test_case bdz +bdz_head() +{ + atf_set "descr" "Checks bdz algorithm" +} +bdz_body() +{ + for n in 4 32 128 1024 65536 131072; do + seq 0 $(($n - 1)) > reference.txt + atf_check -o file:reference.txt \ + $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 bdz "sort -n" \ + $n $(atf_get_srcdir)/hash_driver.c + atf_check -o file:hash.map \ + $(atf_get_srcdir)/h_nbperf /usr/share/dict/web2 bdz cat \ + $n $(atf_get_srcdir)/hash_driver.c + done +} +bdz_clean() +{ + cleanup +} + atf_init_test_cases() { atf_add_test_case chm atf_add_test_case chm3 + atf_add_test_case bdz } Index: src/usr.bin/nbperf/nbperf-bdz.c diff -u src/usr.bin/nbperf/nbperf-bdz.c:1.4 src/usr.bin/nbperf/nbperf-bdz.c:1.5 --- src/usr.bin/nbperf/nbperf-bdz.c:1.4 Fri Oct 21 23:47:11 2011 +++ src/usr.bin/nbperf/nbperf-bdz.c Tue Sep 25 20:53:46 2012 @@ -1,6 +1,6 @@ -/* $NetBSD: nbperf-bdz.c,v 1.4 2011/10/21 23:47:11 joerg Exp $ */ +/* $NetBSD: nbperf-bdz.c,v 1.5 2012/09/25 20:53:46 joerg Exp $ */ /*- - * Copyright (c) 2009 The NetBSD Foundation, Inc. + * Copyright (c) 2009, 2012 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -36,7 +36,7 @@ #endif #include <sys/cdefs.h> -__RCSID("$NetBSD: nbperf-bdz.c,v 1.4 2011/10/21 23:47:11 joerg Exp $"); +__RCSID("$NetBSD: nbperf-bdz.c,v 1.5 2012/09/25 20:53:46 joerg Exp $"); #include <err.h> #include <inttypes.h> @@ -72,10 +72,7 @@ struct state { struct graph3 graph; uint32_t *visited; uint32_t *holes64k; - uint16_t *holes256; - uint8_t *holes256_64; - uint8_t *holes256_128; - uint8_t *holes256_192; + uint16_t *holes64; uint8_t *g; uint32_t *result_map; }; @@ -123,17 +120,8 @@ assign_nodes(struct state *state) if (i % 65536 == 0) state->holes64k[i >> 16] = holes; - if (i % 256 == 0) - state->holes256[i >> 8] = holes - state->holes64k[i >> 16]; - - if (i % 256 == 64) - state->holes256_64[i >> 8] = holes - state->holes256[i >> 8] - state->holes64k[i >> 16]; - - if (i % 256 == 128) - state->holes256_128[i >> 8] = holes - state->holes256[i >> 8] - state->holes64k[i >> 16]; - - if (i % 256 == 192) - state->holes256_192[i >> 8] = holes - state->holes256[i >> 8] - state->holes64k[i >> 16]; + if (i % 64 == 0) + state->holes64[i >> 6] = holes - state->holes64k[i >> 16]; if (state->visited[i] > 1) { j = state->visited[i] - 2; @@ -143,28 +131,13 @@ assign_nodes(struct state *state) if (state->g[i] == 3) ++holes; } - - if (i % 65536 != 0) - state->holes64k[(i >> 16) + 1] = holes; - - if (i % 256 != 0) - state->holes256[(i >> 8) + 1] = holes - state->holes64k[((i >> 8) + 1) >> 8]; - - if (i % 256 != 64) - state->holes256_64[(i >> 8) + 1] = holes - state->holes256[(i >> 8) + 1] - state->holes64k[((i >> 8) + 1) >> 8]; - - if (i % 256 != 128) - state->holes256_128[(i >> 8) + 1] = holes - state->holes256[(i >> 8) + 1] - state->holes64k[((i >> 8) + 1) >> 8]; - - if (i % 256 != 192) - state->holes256_192[(i >> 8) + 1] = holes - state->holes256[(i >> 8) + 1] - state->holes64k[((i >> 8) + 1) >> 8]; } static void print_hash(struct nbperf *nbperf, struct state *state) { - size_t i, j; - uint32_t sum; + uint64_t sum; + size_t i; fprintf(nbperf->output, "#include <stdlib.h>\n"); fprintf(nbperf->output, "#include <strings.h>\n\n"); @@ -175,19 +148,50 @@ print_hash(struct nbperf *nbperf, struct "%s(const void * __restrict key, size_t keylen)\n", nbperf->hash_name); fprintf(nbperf->output, "{\n"); + fprintf(nbperf->output, - "\tstatic const uint32_t g[%" PRId32 "] = {\n", - (state->graph.v + 15) / 16); - for (i = 0; i < state->graph.v; i += 16) { - for (j = 0, sum = 0; j < 16; ++j) - sum |= (uint32_t)state->g[i + j] << (2 * j); + "\tstatic const uint64_t g1[%" PRId32 "] = {\n", + (state->graph.v + 63) / 64); + sum = 0; + for (i = 0; i < state->graph.v; ++i) { + sum |= ((uint64_t)state->g[i] & 1) << (i & 63); + if (i % 64 == 63) { + fprintf(nbperf->output, "%s0x%016" PRIx64 "ULL,%s", + (i / 64 % 2 == 0 ? "\t " : " "), + sum, + (i / 64 % 2 == 1 ? "\n" : "")); + sum = 0; + } + } + if (i % 64 != 0) { + fprintf(nbperf->output, "%s0x%016" PRIx64 "ULL,%s", + (i / 64 % 2 == 0 ? "\t " : " "), + sum, + (i / 64 % 2 == 1 ? "\n" : "")); + } + fprintf(nbperf->output, "%s\t};\n", (i % 2 ? "\n" : "")); - fprintf(nbperf->output, "%s0x%08" PRIx32 "ULL,%s", - (i / 16 % 4 == 0 ? "\t " : " "), + fprintf(nbperf->output, + "\tstatic const uint64_t g2[%" PRId32 "] = {\n", + (state->graph.v + 63) / 64); + sum = 0; + for (i = 0; i < state->graph.v; ++i) { + sum |= (((uint64_t)state->g[i] & 2) >> 1) << (i & 63); + if (i % 64 == 63) { + fprintf(nbperf->output, "%s0x%016" PRIx64 "ULL,%s", + (i / 64 % 2 == 0 ? "\t " : " "), + sum, + (i / 64 % 2 == 1 ? "\n" : "")); + sum = 0; + } + } + if (i % 64 != 0) { + fprintf(nbperf->output, "%s0x%016" PRIx64 "ULL,%s", + (i / 64 % 2 == 0 ? "\t " : " "), sum, - (i / 16 % 4 == 3 ? "\n" : "")); + (i / 64 % 2 == 1 ? "\n" : "")); } - fprintf(nbperf->output, "%s\t};\n", (i / 16 % 4 ? "\n" : "")); + fprintf(nbperf->output, "%s\t};\n", (i % 2 ? "\n" : "")); fprintf(nbperf->output, "\tstatic const uint32_t holes64k[%" PRId32 "] = {\n", @@ -200,111 +204,45 @@ print_hash(struct nbperf *nbperf, struct fprintf(nbperf->output, "%s\t};\n", (i / 65536 % 4 ? "\n" : "")); fprintf(nbperf->output, - "\tstatic const uint16_t holes256[%" PRId32 "] = {\n", - (state->graph.v + 255) / 256); - for (i = 0; i < state->graph.v; i += 256) + "\tstatic const uint16_t holes64[%" PRId32 "] = {\n", + (state->graph.v + 63) / 64); + for (i = 0; i < state->graph.v; i += 64) fprintf(nbperf->output, "%s0x%04" PRIx32 ",%s", - (i / 256 % 4 == 0 ? "\t " : " "), - state->holes256[i >> 8], - (i / 256 % 4 == 3 ? "\n" : "")); - fprintf(nbperf->output, "%s\t};\n", (i / 256 % 4 ? "\n" : "")); - - fprintf(nbperf->output, - "\tstatic const uint8_t holes256_64[%" PRId32 "] = {\n", - (state->graph.v + 255) / 256); - for (i = 64; i < state->graph.v; i += 256) - fprintf(nbperf->output, "%s0x%02" PRIx32 ",%s", - (i / 256 % 4 == 0 ? "\t " : " "), - state->holes256_64[i >> 8], - (i / 256 % 4 == 3 ? "\n" : "")); - fprintf(nbperf->output, "%s\t};\n", (i / 256 % 4 ? "\n" : "")); - - fprintf(nbperf->output, - "\tstatic const uint8_t holes256_128[%" PRId32 "] = {\n", - (state->graph.v + 255) / 256); - for (i = 128; i < state->graph.v; i += 256) - fprintf(nbperf->output, "%s0x%02" PRIx32 ",%s", - (i / 256 % 4 == 0 ? "\t " : " "), - state->holes256_128[i >> 8], - (i / 256 % 4 == 3 ? "\n" : "")); - fprintf(nbperf->output, "%s\t};\n", (i / 256 % 4 ? "\n" : "")); - - fprintf(nbperf->output, - "\tstatic const uint8_t holes256_192[%" PRId32 "] = {\n", - (state->graph.v + 255) / 256); - for (i = 192; i < state->graph.v; i += 256) - fprintf(nbperf->output, "%s0x%02" PRIx32 ",%s", - (i / 256 % 4 == 0 ? "\t " : " "), - state->holes256_192[i >> 8], - (i / 256 % 4 == 3 ? "\n" : "")); - fprintf(nbperf->output, "%s\t};\n", (i / 256 % 4 ? "\n" : "")); + (i / 64 % 4 == 0 ? "\t " : " "), + state->holes64[i >> 6], + (i / 64 % 4 == 3 ? "\n" : "")); + fprintf(nbperf->output, "%s\t};\n", (i / 64 % 4 ? "\n" : "")); + fprintf(nbperf->output, "\tuint64_t m;\n"); + fprintf(nbperf->output, "\tuint32_t idx, i, idx2;\n"); fprintf(nbperf->output, "\tuint32_t h[%zu];\n\n", nbperf->hash_size); - fprintf(nbperf->output, "\tuint32_t m;\n"); - fprintf(nbperf->output, "\tuint32_t a1, a2, b1, b2, c1, c2, idx, idx2;\n\n"); (*nbperf->print_hash)(nbperf, "\t", "key", "keylen", "h"); - fprintf(nbperf->output, "\n\th[0] = h[0] %% %" PRIu32 ";\n", state->graph.v); - fprintf(nbperf->output, "\th[1] = h[1] %% %" PRIu32 ";\n", state->graph.v); - fprintf(nbperf->output, "\th[2] = h[2] %% %" PRIu32 ";\n", state->graph.v); - - fprintf(nbperf->output, "\n\ta1 = h[0] >> 4;\n"); - fprintf(nbperf->output, "\ta2 = 2 * (h[0] & 15);\n"); - fprintf(nbperf->output, "\tb1 = h[1] >> 4;\n"); - fprintf(nbperf->output, "\tb2 = 2 * (h[1] & 15);\n"); - fprintf(nbperf->output, "\tc1 = h[2] >> 4;\n"); - fprintf(nbperf->output, "\tc2 = 2 * (h[2] & 15);\n"); + fprintf(nbperf->output, "\n\th[0] = h[0] %% %" PRIu32 ";\n", + state->graph.v); + fprintf(nbperf->output, "\th[1] = h[1] %% %" PRIu32 ";\n", + state->graph.v); + fprintf(nbperf->output, "\th[2] = h[2] %% %" PRIu32 ";\n", + state->graph.v); fprintf(nbperf->output, - "\tidx = h[(((g[a1] >> a2) & 3) + ((g[b1] >> b2) & 3) +\n" - "\t ((g[c1] >> c2) & 3)) %% 3];\n\n"); + "\tidx = 9 + ((g1[h[0] >> 6] >> (h[0] & 63)) &1)" + "\t + ((g1[h[1] >> 6] >> (h[1] & 63)) & 1)" + "\t + ((g1[h[2] >> 6] >> (h[2] & 63)) & 1)" + "\t - ((g2[h[0] >> 6] >> (h[0] & 63)) & 1)" + "\t - ((g2[h[1] >> 6] >> (h[1] & 63)) & 1)" + "\t - ((g2[h[2] >> 6] >> (h[2] & 63)) & 1);" + ); fprintf(nbperf->output, - "\tswitch ((idx >> 5) & 7) {\n" - "\tcase 0:\n" - "\t\tidx2 = idx - holes64k[idx >> 16] - holes256[idx >> 8];\n" - "\t\tbreak;\n" - "\tcase 1: case 2:\n" - "\t\tidx2 = idx - holes64k[idx >> 16] - holes256[idx >> 8]\n" - "\t\t - holes256_64[idx >> 8];\n" - "\t\tbreak;\n" - "\tcase 3: case 4:\n" - "\t\tidx2 = idx - holes64k[idx >> 16] - holes256[idx >> 8]\n" - "\t\t - holes256_128[idx >> 8];\n" - "\t\tbreak;\n" - "\tcase 5: case 6:\n" - "\t\tidx2 = idx - holes64k[idx >> 16] - holes256[idx >> 8]\n" - "\t\t - holes256_192[idx >> 8];\n" - "\t\tbreak;\n" - "\tcase 7:\n" - "\t\tidx2 = idx - holes64k[(idx + 32) >> 16] -\n" - "\t\t holes256[(idx + 32) >> 8];\n" - "\t\tbreak;\n" - "\tdefault:\n" - "\t\tabort();\n" - "\t}\n" - "\tswitch ((idx >> 4) & 3) {\n" - "\tcase 1:\n" - "\t\tm = (g[(idx >> 4) - 1] & (g[(idx >> 4) - 1] >> 1) & 0x55555555U);\n" - "\t\tidx2 -= popcount32(m);\n" - "\tcase 0:\n" - "\t\tm = (g[idx >> 4] & (g[idx >> 4] >> 1) & 0x55555555U);\n" - "\t\tm &= ((2U << (2 * (idx & 15))) - 1);\n" - "\t\tidx2 -= popcount32(m);\n" - "\t\tbreak;\n" - "\tcase 2:\n" - "\t\tm = (g[(idx >> 4) + 1] & (g[(idx >> 4) + 1] >> 1) & 0x55555555U);\n" - "\t\tidx2 += popcount32(m);\n" - "\tcase 3:\n" - "\t\tm = (g[idx >> 4] & (g[idx >> 4] >> 1) & 0x55555555U);\n" - "\t\tm &= ~((2U << (2 * (idx & 15))) - 1);\n" - "\t\tidx2 += popcount32(m);\n" - "\t\tbreak;\n" - "\t}\n\n"); - + "\tidx = h[idx %% 3];\n"); fprintf(nbperf->output, - "\treturn idx2;\n"); + "\tidx2 = idx - holes64[idx >> 6] - holes64k[idx >> 16];\n" + "\tidx2 -= popcount64(g1[idx >> 6] & g2[idx >> 6]\n" + "\t & (((uint64_t)1 << idx) - 1));\n" + "\treturn idx2;"); + fprintf(nbperf->output, "}\n"); if (nbperf->map_output != NULL) { @@ -338,19 +276,15 @@ bdz_compute(struct nbperf *nbperf) graph3_setup(&state.graph, v, e); - state.holes64k = calloc(sizeof(uint32_t), (v + 65535) / 65536 + 1); - state.holes256 = calloc(sizeof(uint16_t), (v + 255) / 256 + 1); - state.holes256_64 = calloc(sizeof(uint8_t), (v + 255) / 256 + 1); - state.holes256_128 = calloc(sizeof(uint8_t), (v + 255) / 256 + 1); - state.holes256_192 = calloc(sizeof(uint8_t), (v + 255) / 256 + 1); - state.g = calloc(sizeof(uint32_t), v); + state.holes64k = calloc(sizeof(uint32_t), (v + 65535) / 65536); + state.holes64 = calloc(sizeof(uint16_t), (v + 63) / 64 ); + state.g = calloc(sizeof(uint32_t), v | 63); state.visited = calloc(sizeof(uint32_t), v); state.result_map = calloc(sizeof(uint32_t), e); - if (state.holes64k == NULL || state.holes256 == NULL || - state.holes256_64 == NULL || state.holes256_128 == NULL || - state.holes256_192 == NULL || state.g == NULL || - state.visited == NULL || state.result_map == NULL) + if (state.holes64k == NULL || state.holes64 == NULL || + state.g == NULL || state.visited == NULL || + state.result_map == NULL) err(1, "malloc failed"); if (graph3_hash(nbperf, &state.graph)) @@ -367,10 +301,7 @@ failed: free(state.visited); free(state.g); free(state.holes64k); - free(state.holes256); - free(state.holes256_64); - free(state.holes256_128); - free(state.holes256_192); + free(state.holes64); free(state.result_map); return retval; } Index: src/usr.bin/nbperf/nbperf.1 diff -u src/usr.bin/nbperf/nbperf.1:1.4 src/usr.bin/nbperf/nbperf.1:1.5 --- src/usr.bin/nbperf/nbperf.1:1.4 Thu May 31 21:36:06 2012 +++ src/usr.bin/nbperf/nbperf.1 Tue Sep 25 20:53:46 2012 @@ -1,4 +1,4 @@ -.\" $NetBSD: nbperf.1,v 1.4 2012/05/31 21:36:06 joerg Exp $ +.\" $NetBSD: nbperf.1,v 1.5 2012/09/25 20:53:46 joerg Exp $ .\" .\" Copyright (c) 2009 The NetBSD Foundation, Inc. .\" All rights reserved. @@ -27,7 +27,7 @@ .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE .\" POSSIBILITY OF SUCH DAMAGE. .\" -.Dd May 31, 2012 +.Dd September 25, 2012 .Dt NBPERF 1 .Os .Sh NAME @@ -90,7 +90,7 @@ noticable smaller than the output for .Ar chm . .It Sy bdz This results in a non-order preserving minimal perfect hash function. -Output size is approximately 2.85 bit per key for the default value of +Output size is approximately 2.79 bit per key for the default value of .Ar utilisation , 1.24. This is also the smallest supported value.