Changeset: 828777a0ed76 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/828777a0ed76
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
sql/backends/monet5/UDF/pyapi3/pyapi3.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/storage/store.c
Branch: resource_management
Log Message:
merge default
diffs (truncated from 1831 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -49879,6 +49879,11 @@ unsafe pattern sql.transaction_rollback(
SQLtransaction_rollback;
A transaction statement (type can be commit,release,rollback or start)
sql
+unclosed_result_sets
+pattern sql.unclosed_result_sets() (X_0:bat[:oid], X_1:bat[:int])
+sql_unclosed_result_sets;
+return query_id/res_id of unclosed result sets
+sql
unionfunc
pattern sql.unionfunc(X_0:str, X_1:str, X_2:any...):any...
SQLunionfunc;
diff --git a/clients/Tests/MAL-signatures.test
b/clients/Tests/MAL-signatures.test
--- a/clients/Tests/MAL-signatures.test
+++ b/clients/Tests/MAL-signatures.test
@@ -38274,6 +38274,11 @@ unsafe pattern sql.transaction_rollback(
SQLtransaction_rollback;
A transaction statement (type can be commit,release,rollback or start)
sql
+unclosed_result_sets
+pattern sql.unclosed_result_sets() (X_0:bat[:oid], X_1:bat[:int])
+sql_unclosed_result_sets;
+return query_id/res_id of unclosed result sets
+sql
unionfunc
pattern sql.unionfunc(X_0:str, X_1:str, X_2:any...):any...
SQLunionfunc;
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -351,6 +351,7 @@ void *GDKzalloc(size_t size) __attribute
void HASHdestroy(BAT *b);
BUN HASHlist(Hash *h, BUN i);
BUN HASHprobe(const Hash *h, const void *v);
+size_t HASHsize(BAT *b);
void HEAP_free(Heap *heap, var_t block);
gdk_return HEAP_initialize(Heap *heap, size_t nbytes, size_t nprivate, int
alignment);
var_t HEAP_malloc(BAT *b, size_t nbytes);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -874,7 +874,6 @@ typedef struct {
/* assert that atom width is power of 2, i.e., width == 1<<shift */
#define assert_shift_width(shift,width) assert(((shift) == 0 && (width) == 0)
|| ((unsigned)1<<(shift)) == (unsigned)(width))
-#define GDKLIBRARY_TAILN 061043U /* first in Jul2021: str offset heaps
names don't take width into account */
#define GDKLIBRARY_HASHASH 061044U /* first in Jul2021: hashash bit in
string heaps */
#define GDKLIBRARY_HSIZE 061045U /* first in Jan2022: heap "size" values
*/
#define GDKLIBRARY_JSON 061046U /* first in Sep2022: json storage
changes*/
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -906,26 +906,10 @@ BBPcheckbats(unsigned bbpversion)
/* first check string offset heap with width,
* then without */
if (MT_stat(path, &statb) < 0) {
-#ifdef GDKLIBRARY_TAILN
- if (b->ttype == TYPE_str &&
- b->twidth < SIZEOF_VAR_T) {
- size_t taillen = strlen(path) - 1;
- char tailsave = path[taillen];
- path[taillen] = 0;
- if (MT_stat(path, &statb) < 0) {
- GDKsyserror("cannot stat file
%s%c or %s (expected size %zu)\n",
- path, tailsave,
path, b->theap->free);
- GDKfree(path);
- return GDK_FAIL;
- }
- } else
-#endif
- {
- GDKsyserror("cannot stat file %s
(expected size %zu)\n",
- path, b->theap->free);
- GDKfree(path);
- return GDK_FAIL;
- }
+ GDKsyserror("cannot stat file %s (expected size
%zu)\n",
+ path, b->theap->free);
+ GDKfree(path);
+ return GDK_FAIL;
}
if ((size_t) statb.st_size < b->theap->free) {
GDKerror("file %s too small (expected %zu,
actual %zu)\n", path, b->theap->free, (size_t) statb.st_size);
@@ -1007,8 +991,7 @@ BBPheader(FILE *fp, int *lineno, bat *bb
bbpversion != GDKLIBRARY_STATUS &&
bbpversion != GDKLIBRARY_JSON &&
bbpversion != GDKLIBRARY_HSIZE &&
- bbpversion != GDKLIBRARY_HASHASH &&
- bbpversion != GDKLIBRARY_TAILN) {
+ bbpversion != GDKLIBRARY_HASHASH) {
TRC_CRITICAL(GDK, "incompatible BBP version: expected 0%o, got
0%o. "
"This database was probably created by a %s
version of MonetDB.",
GDKLIBRARY, bbpversion,
@@ -1424,56 +1407,6 @@ fixhashash(bat *hashbats, bat nhashbats)
}
#endif
-#ifdef GDKLIBRARY_TAILN
-static gdk_return
-movestrbats(void)
-{
- for (bat bid = 1, nbat = (bat) ATOMIC_GET(&BBPsize); bid < nbat; bid++)
{
- BAT *b = BBP_desc(bid);
- if (b->batCacheid == 0) {
- /* not a valid BAT */
- continue;
- }
- if (b->ttype != TYPE_str || b->twidth == SIZEOF_VAR_T ||
b->batCount == 0)
- continue;
- char *oldpath = GDKfilepath(0, BATDIR,
BBP_physical(b->batCacheid), "tail");
- char *newpath = GDKfilepath(0, BATDIR, b->theap->filename,
NULL);
- int ret = -1;
- if (oldpath != NULL && newpath != NULL) {
- struct stat oldst, newst;
- bool oldexist = MT_stat(oldpath, &oldst) == 0;
- bool newexist = MT_stat(newpath, &newst) == 0;
- if (newexist) {
- if (oldexist) {
- if (oldst.st_mtime > newst.st_mtime) {
- GDKerror("both %s and %s exist
with %s unexpectedly newer: manual intervention required\n", oldpath, newpath,
oldpath);
- ret = -1;
- } else {
- GDKwarning("both %s and %s
exist, removing %s\n", oldpath, newpath, oldpath);
- ret = MT_remove(oldpath);
- }
- } else {
- /* already good */
- ret = 0;
- }
- } else if (oldexist) {
- TRC_DEBUG(IO_, "rename %s to %s\n", oldpath,
newpath);
- ret = MT_rename(oldpath, newpath);
- } else {
- /* neither file exists: may be ok, but
- * will be checked later */
- ret = 0;
- }
- }
- GDKfree(oldpath);
- GDKfree(newpath);
- if (ret == -1)
- return GDK_FAIL;
- }
- return GDK_SUCCEED;
-}
-#endif
-
#ifdef GDKLIBRARY_JSON
static gdk_return
jsonupgradebat(BAT *b, json_storage_conversion fixJSONStorage)
@@ -2013,56 +1946,6 @@ BBPinit(bool allow_hge_upgrade)
return GDK_FAIL;
}
-#ifdef GDKLIBRARY_TAILN
- char *needstrbatmove;
- if (GDKinmemory(0)) {
- needstrbatmove = NULL;
- } else {
- if ((needstrbatmove = GDKfilepath(0, BATDIR, "needstrbatmove",
NULL)) == NULL) {
-#ifdef GDKLIBRARY_HASHASH
- GDKfree(hashbats);
-#endif
- ATOMIC_SET(&GDKdebug, dbg);
- return GDK_FAIL;
- }
- if (bbpversion <= GDKLIBRARY_TAILN) {
- /* create signal file that we need to rename string
- * offset heaps */
- int fd = MT_open(needstrbatmove, O_WRONLY | O_CREAT);
- if (fd < 0) {
- TRC_CRITICAL(GDK, "cannot create signal file
needstrbatmove.\n");
- GDKfree(needstrbatmove);
-#ifdef GDKLIBRARY_HASHASH
- GDKfree(hashbats);
-#endif
- ATOMIC_SET(&GDKdebug, dbg);
- return GDK_FAIL;
- }
- close(fd);
- } else {
- /* check signal file whether we need to rename string
- * offset heaps */
- int fd = MT_open(needstrbatmove, O_RDONLY);
- if (fd >= 0) {
- /* yes, we do */
- close(fd);
- } else if (errno == ENOENT) {
- /* no, we don't: set var to NULL */
- GDKfree(needstrbatmove);
- needstrbatmove = NULL;
- } else {
- GDKsyserror("unexpected error opening %s\n",
needstrbatmove);
- GDKfree(needstrbatmove);
-#ifdef GDKLIBRARY_HASHASH
- GDKfree(hashbats);
-#endif
- ATOMIC_SET(&GDKdebug, dbg);
- return GDK_FAIL;
- }
- }
- }
-#endif
-
#ifdef GDKLIBRARY_HASHASH
if (nhashbats > 0)
res = fixhashash(hashbats, nhashbats);
@@ -2108,25 +1991,6 @@ BBPinit(bool allow_hge_upgrade)
return GDK_FAIL;
}
-#ifdef GDKLIBRARY_TAILN
- /* we rename the offset heaps after the above commit: in this
- * version we accept both the old and new names, but we want to
- * convert so that future versions only have the new name */
- if (needstrbatmove) {
- /* note, if renaming fails, nothing is lost: a next
- * invocation will just try again; an older version of
- * mserver will not work because of the TMcommit
- * above */
- if (movestrbats() != GDK_SUCCEED) {
- GDKfree(needstrbatmove);
- ATOMIC_SET(&GDKdebug, dbg);
- return GDK_FAIL;
- }
- MT_remove(needstrbatmove);
- GDKfree(needstrbatmove);
- needstrbatmove = NULL;
- }
-#endif
ATOMIC_SET(&GDKdebug, dbg);
/* cleanup any leftovers (must be done after BBPrecover) */
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -562,6 +562,40 @@ BATcheckhash(BAT *b)
return h != NULL;
}
+/* figure out size of the hash (sum of the sizes of the two hash files)
+ * without loading them */
+size_t
+HASHsize(BAT *b)
+{
+ size_t sz = 0;
+ MT_rwlock_rdlock(&b->thashlock);
+ if (b->thash == NULL) {
+ sz = 0;
+ } else if (b->thash != (Hash *) 1) {
+ sz = b->thash->heaplink.size + b->thash->heapbckt.size;
+ } else {
+ int farmid = BBPselectfarm(b->batRole, b->ttype, hashheap);
+ if (farmid >= 0) {
+ const char *nme = BBP_physical(b->batCacheid);
+ char *fname = GDKfilepath(farmid, BATDIR, nme,
"thashb");
+ if (fname != NULL) {
+ struct stat st;
+ if (stat(fname, &st) == 0) {
+ sz = (size_t) st.st_size;
+ fname[strlen(fname) - 1] = 'l';
+ if (stat(fname, &st) == 0)
+ sz += (size_t) st.st_size;
+ else
+ sz = 0;
+ }
+ GDKfree(fname);
+ }
+ }
+ }
+ MT_rwlock_rdunlock(&b->thashlock);
+ return sz;
+}
+
static void
BAThashsave_intern(BAT *b, bool dosync)
{
diff --git a/gdk/gdk_hash.h b/gdk/gdk_hash.h
--- a/gdk/gdk_hash.h
+++ b/gdk/gdk_hash.h
@@ -37,6 +37,7 @@ gdk_export gdk_return BAThash(BAT *b);
gdk_export void HASHdestroy(BAT *b);
gdk_export BUN HASHprobe(const Hash *h, const void *v);
gdk_export BUN HASHlist(Hash *h, BUN i);
+gdk_export size_t HASHsize(BAT *b);
#define BUN2 2
#define BUN4 4
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -95,7 +95,7 @@ HEAPgrow(Heap **hp, size_t size, bool ma
ATOMIC_BASE_TYPE refs = ATOMIC_GET(&(*hp)->refs);
if ((refs & HEAPREFS) == 1) {
- return HEAPextend((*hp), size, mayshare);
+ return HEAPextend(*hp, size, mayshare);
_______________________________________________
checkin-list mailing list -- checkin-list@monetdb.org
To unsubscribe send an email to checkin-list-le...@monetdb.org
