Changeset: 6920e02d734f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6920e02d734f
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_cand.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_orderidx.c
gdk/gdk_private.h
gdk/gdk_string.c
monetdb5/modules/mal/tablet.c
Branch: unlock
Log Message:
Convert string offset heaps to var_t size when starting to mmap them.
This avoids the problem that a view might be looking at at (mmapped)
offset heap that gets widened by the parent when it inserts extra data.
diffs (truncated from 807 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -323,7 +323,7 @@ gdk_return GDKtracer_set_flush_level(con
gdk_return GDKtracer_set_layer_level(const char *layer, const char *lvl);
gdk_return GDKtracer_stop(void);
size_t GDKuniqueid(size_t offset);
-gdk_return GDKupgradevarheap(BAT *b, var_t v, bool copyall, bool mayshare)
__attribute__((__warn_unused_result__));
+gdk_return GDKupgradevarheap(BAT *b, var_t v, BUN cap, bool copyall)
__attribute__((__warn_unused_result__));
lng GDKusec(void);
const char *GDKversion(void) __attribute__((__const__));
size_t GDKvm_cursize(void);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -889,7 +889,7 @@ gdk_export gdk_return BATextend(BAT *b,
gdk_export uint8_t ATOMelmshift(int sz)
__attribute__((__const__));
-gdk_export gdk_return GDKupgradevarheap(BAT *b, var_t v, bool copyall, bool
mayshare)
+gdk_export gdk_return GDKupgradevarheap(BAT *b, var_t v, BUN cap, bool copyall)
__attribute__((__warn_unused_result__));
gdk_export gdk_return BUNappend(BAT *b, const void *right, bool force)
__attribute__((__warn_unused_result__));
@@ -1507,8 +1507,7 @@ Tputvalue(BAT *b, BUN p, const void *v,
if (b->twidth < SIZEOF_VAR_T &&
(b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 <<
(8 * b->twidth))) {
/* doesn't fit in current heap, upgrade it */
- rc = GDKupgradevarheap(b, d, copyall,
- b->batRestricted == BAT_READ);
+ rc = GDKupgradevarheap(b, d, 0, copyall);
if (rc != GDK_SUCCEED)
return rc;
}
@@ -1592,8 +1591,7 @@ tfastins_nocheckVAR(BAT *b, BUN p, const
if (b->twidth < SIZEOF_VAR_T &&
(b->twidth <= 2 ? d - GDK_VAROFFSET : d) >= ((size_t) 1 << (8 *
b->twidth))) {
/* doesn't fit in current heap, upgrade it */
- rc = GDKupgradevarheap(b, d, false,
- b->batRestricted == BAT_READ);
+ rc = GDKupgradevarheap(b, d, 0, false);
if (rc != GDK_SUCCEED)
return rc;
}
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -187,7 +187,7 @@ BATmaterialize(BAT *b)
strconcat_len(b->theap->filename, sizeof(b->theap->filename),
BBP_physical(b->batCacheid), ".tail", NULL);
- if (HEAPalloc(tail, cnt, sizeof(oid)) != GDK_SUCCEED) {
+ if (HEAPalloc(tail, cnt, sizeof(oid), 0) != GDK_SUCCEED) {
return GDK_FAIL;
}
@@ -196,6 +196,7 @@ BATmaterialize(BAT *b)
assert(ATOMIC_GET(&b->theap->refs) > 0);
HEAPdecref(b->theap, false);
b->theap = tail;
+ b->tbaseoff = 0;
MT_lock_unset(&b->theaplock);
b->ttype = TYPE_oid;
BATsetdims(b);
@@ -318,7 +319,7 @@ VIEWreset(BAT *b)
strconcat_len(tail->filename, sizeof(tail->filename),
nme, ".tail", NULL);
- if (b->ttype && HEAPalloc(tail, cnt, Tsize(b)) != GDK_SUCCEED)
+ if (b->ttype && HEAPalloc(tail, cnt, Tsize(b),
ATOMsize(b->ttype)) != GDK_SUCCEED)
goto bailout;
if (b->tvheap) {
th = GDKmalloc(sizeof(Heap));
@@ -363,6 +364,10 @@ VIEWreset(BAT *b)
b->tkey = BATtkey(v);
/* replace the heaps */
+ if (tail->storage == STORE_MMAP) {
+ b->twidth = ATOMsize(b->ttype);
+ b->tshift = ATOMelmshift(Tsize(b));
+ }
b->theap = tail;
tail = NULL;
b->tbaseoff = 0;
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -213,9 +213,13 @@ COLnew(oid hseq, int tt, BUN cap, role_t
bn->batCapacity = cap;
/* alloc the main heaps */
- if (tt && HEAPalloc(bn->theap, cap, bn->twidth) != GDK_SUCCEED) {
+ if (tt && HEAPalloc(bn->theap, cap, bn->twidth, ATOMsize(bn->ttype)) !=
GDK_SUCCEED) {
goto bailout;
}
+ if (bn->theap->storage == STORE_MMAP) {
+ bn->twidth = ATOMsize(bn->ttype);
+ bn->tshift = ATOMelmshift(Tsize(bn));
+ }
if (bn->tvheap && ATOMheap(tt, bn->tvheap, cap) != GDK_SUCCEED) {
goto bailout;
@@ -469,6 +473,8 @@ BATextend(BAT *b, BUN newcap)
if (b->theap->base) {
TRC_DEBUG(HEAP, "HEAPgrow in BATextend %s %zu %zu\n",
b->theap->filename, b->theap->size, theap_size);
+ if (b->ttype == TYPE_str)
+ return GDKupgradevarheap(b, GDK_VAROFFSET, newcap,
false);
Heap *h = HEAPgrow(b->theap, theap_size);
if (h == NULL)
return GDK_FAIL;
@@ -1094,6 +1100,8 @@ BUNappend(BAT *b, const void *t, bool fo
b->tseqbase = * (const oid *) t;
} else if (is_oid_nil(* (oid *) t) ||
b->tseqbase + b->batCount != *(const oid *) t) {
+ /* we need to materialize b; allocate enough capacity */
+ b->batCapacity = BATcount(b) + 1;
if (BATmaterialize(b) != GDK_SUCCEED)
return GDK_FAIL;
}
@@ -1334,7 +1342,7 @@ BUNinplace(BAT *b, BUN p, const void *t,
if (b->twidth < SIZEOF_VAR_T &&
(b->twidth <= 2 ? _d - GDK_VAROFFSET : _d) >= ((size_t) 1
<< (8 * b->twidth))) {
/* doesn't fit in current heap, upgrade it */
- if (GDKupgradevarheap(b, _d, false, b->batRestricted ==
BAT_READ) != GDK_SUCCEED)
+ if (GDKupgradevarheap(b, _d, 0, false) != GDK_SUCCEED)
return GDK_FAIL;
}
_ptr = BUNtloc(bi, p);
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -57,7 +57,7 @@ unshare_string_heap(BAT *b)
#endif
static gdk_return
-insert_string_bat(BAT *b, BAT *n, struct canditer *ci, bool force)
+insert_string_bat(BAT *b, BAT *n, struct canditer *ci)
{
BATiter ni; /* iterator */
size_t toff = ~(size_t) 0; /* tail offset */
@@ -193,26 +193,15 @@ insert_string_bat(BAT *b, BAT *n, struct
}
}
}
- if (toff != ~(size_t) 0) {
- /* we only have to copy the offsets from n to
- * b, possibly with an offset (if toff != 0),
- * so set up some variables and set up b's
- * tail so that it looks like it's a fixed
- * size column. Of course, we must make sure
- * first that the width of b's offset heap can
- * accommodate all values. */
- if (b->twidth < SIZEOF_VAR_T &&
- ((size_t) 1 << 8 * b->twidth) <= (b->twidth <= 2 ?
b->tvheap->size - GDK_VAROFFSET : b->tvheap->size)) {
- /* offsets aren't going to fit, so
- * widen offset heap */
- if (GDKupgradevarheap(b, (var_t)
b->tvheap->size, false, force) != GDK_SUCCEED) {
- toff = ~(size_t) 0;
- return GDK_FAIL;
- }
- }
- }
} else if (unshare_string_heap(b) != GDK_SUCCEED)
return GDK_FAIL;
+
+ /* make sure there is (vertical) space in the offset heap, we
+ * may have to widen the heap later */
+ if (GDKupgradevarheap(b, (var_t) b->tvheap->size, BATcount(b) + cnt,
+ false) != GDK_SUCCEED)
+ return GDK_FAIL;
+
if (toff == 0 && n->twidth == b->twidth && ci->tpe == cand_dense) {
/* we don't need to do any translation of offset
* values, so we can use fast memcpy */
@@ -224,6 +213,7 @@ insert_string_bat(BAT *b, BAT *n, struct
* string heap at known locations (namely the offset
* in n added to toff), so insert offsets from n after
* adding toff into b */
+
/* note the use of the "restrict" qualifier here: all
* four pointers below point to the same value, but
* only one of them will actually be used, hence we
@@ -286,14 +276,6 @@ insert_string_bat(BAT *b, BAT *n, struct
v = (var_t) ((size_t) v + toff);
assert(v >= GDK_VAROFFSET);
assert((size_t) v < b->tvheap->free);
- if (BUNlast(b) >= BATcapacity(b)) {
- if (BATcount(b) == BUN_MAX) {
- GDKerror("too many elements to
accomodate (" BUNFMT ")\n", BUN_MAX);
- goto bunins_failed;
- }
- if (BATextend(b, BATgrows(b)) != GDK_SUCCEED)
- goto bunins_failed;
- }
switch (b->twidth) {
case 1:
assert(v - GDK_VAROFFSET < ((var_t) 1 << 8));
@@ -360,14 +342,6 @@ insert_string_bat(BAT *b, BAT *n, struct
* have to insert a new string into b:
* we can just copy the offset */
v = (var_t) off;
- if (b->twidth < SIZEOF_VAR_T &&
- ((size_t) 1 << 8 * b->twidth) <= (b->twidth
<= 2 ? v - GDK_VAROFFSET : v)) {
- /* offset isn't going to fit,
- * so widen offset heap */
- if (GDKupgradevarheap(b, v, false,
force) != GDK_SUCCEED) {
- goto bunins_failed;
- }
- }
switch (b->twidth) {
case 1:
assert(v - GDK_VAROFFSET < ((var_t) 1
<< 8));
@@ -425,6 +399,17 @@ append_varsized_bat(BAT *b, BAT *n, stru
assert(b->twidth == SIZEOF_VAR_T);
if (cnt == 0)
return GDK_SUCCEED;
+ if (cnt > BATcapacity(b) - BATcount(b)) {
+ /* if needed space exceeds a normal growth extend just
+ * with what's needed */
+ BUN ncap = BATcount(b) + cnt;
+ BUN grows = BATgrows(b);
+
+ if (ncap > grows)
+ grows = ncap;
+ if (BATextend(b, grows) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
if (BATcount(b) == 0 &&
b->batRole == TRANSIENT &&
n->batRestricted == BAT_READ &&
@@ -432,9 +417,8 @@ append_varsized_bat(BAT *b, BAT *n, stru
/* if b is still empty, in the transient farm, and n
* is read-only, we replace b's vheap with a reference
* to n's */
- /* make sure locking happens in a
- * predictable order: lowest id
- * first */
+ /* make sure locking happens in a predictable order:
+ * lowest id first */
if (b->batCacheid < n->batCacheid) {
MT_lock_set(&b->theaplock);
MT_lock_set(&n->theaplock);
@@ -532,7 +516,7 @@ BATappend(BAT *b, BAT *n, BAT *s, bool f
char buf[64];
lng t0 = 0;
- if (b == NULL || n == NULL || (cnt = BATcount(n)) == 0) {
+ if (b == NULL || n == NULL || BATcount(n) == 0) {
return GDK_SUCCEED;
}
assert(b->batCacheid > 0);
@@ -638,27 +622,17 @@ BATappend(BAT *b, BAT *n, BAT *s, bool f
b->batCapacity = BATcount(b) + cnt;
if (BATmaterialize(b) != GDK_SUCCEED)
return GDK_FAIL;
- } else if (cnt > BATcapacity(b) - BATcount(b)) {
- /* if needed space exceeds a normal growth extend just
- * with what's needed */
- BUN ncap = BATcount(b) + cnt;
- BUN grows = BATgrows(b);
-
- if (ncap > grows)
- grows = ncap;
- if (BATextend(b, grows) != GDK_SUCCEED)
- return GDK_FAIL;
}
r = BUNlast(b);
+ /* property setting */
if (BATcount(b) == 0) {
b->tsorted = n->tsorted;
b->trevsorted = n->trevsorted;
b->tseqbase = oid_nil;
b->tnonil = n->tnonil;
b->tnil = n->tnil && cnt == BATcount(n);
- b->tseqbase = oid_nil;
if (ci.tpe == cand_dense) {
b->tnosorted = ci.seq - hseq <= n->tnosorted &&
n->tnosorted < ci.seq + cnt - hseq ? n->tnosorted + hseq - ci.seq : 0;
b->tnorevsorted = ci.seq - hseq <= n->tnorevsorted &&
n->tnorevsorted < ci.seq + cnt - hseq ? n->tnorevsorted + hseq - ci.seq : 0;
@@ -707,8 +681,9 @@ BATappend(BAT *b, BAT *n, BAT *s, bool f
b->tnonil &= n->tnonil;
b->tnil |= n->tnil && cnt == BATcount(n);
}
+
if (b->ttype == TYPE_str) {
- if (insert_string_bat(b, n, &ci, force) != GDK_SUCCEED) {
+ if (insert_string_bat(b, n, &ci) != GDK_SUCCEED) {
return GDK_FAIL;
}
} else if (ATOMvarsized(b->ttype)) {
@@ -716,6 +691,17 @@ BATappend(BAT *b, BAT *n, BAT *s, bool f
return GDK_FAIL;
}
} else {
+ if (cnt > BATcapacity(b) - BATcount(b)) {
+ /* if needed space exceeds a normal growth
+ * extend just with what's needed */
+ BUN ncap = BATcount(b) + cnt;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
