Changeset: 8d6fd846cca5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/8d6fd846cca5
Branch: manti
Log Message:
merged with default
diffs (truncated from 9329 to 300 lines):
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -48,7 +48,7 @@ jobs:
- name: install packages
run: |
sudo apt-get update
- sudo apt-get install -y bison libbz2-dev libpcre2-dev liblzma-dev
liblz4-dev libxml2-dev pkg-config libxxhash-dev unixodbc-dev zlib1g-dev
libssl-dev
+ sudo apt-get install -y bison libbz2-dev libpcre2-dev liblzma-dev
liblz4-dev libxml2-dev pkg-config libxxhash-dev unixodbc-dev zlib1g-dev
libssl-dev libhts-dev
if: runner.os == 'Linux'
- name: Patch /etc/hosts on Unix
@@ -74,7 +74,7 @@ jobs:
if: runner.os == 'Linux'
- name: brew packages
- run: brew install bison pkgconf xxhash
+ run: brew install bison pkgconf xxhash htslib
if: runner.os == 'macOS'
- name: make MonetDB on macos
diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,21 @@
# ChangeLog file for devel
# This file is updated with Maddlog
+* Thu Oct 22 2026 Martin van Dinther <[email protected]>
+- Added support for reading DNA sequence alignment data from *.sam and *.bam
files.
+ Usage examples:
+ SELECT * FROM '/path/to/file1.bam';
+ or
+ SELECT * FROM file_loader('/path/to/file1.bam');
+ Prerequisites:
+ We use the HTSlib for reading these files and the library needs to be
+ pre-installed on your OS (e.g. on Fedora via: sudo dnf install htslib).
+ See also: https://www.htslib.org/download/
+ The HTSlib must be version 1.24 or higher.
+ Also the MonetDB server has to be started with startup argument:
+ --loadmodule htslib
+ to enable it. By default it is not enabled.
+
* Wed Oct 21 2026 Sjoerd Mullender <[email protected]>
- There is a new type of database farm, like the --dbextra farm, for
temporary heaps. Temporary heaps are heaps which are created and
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -102,9 +102,9 @@ The following packages are optional but
``readline-devel``, ``xz-devel``, ``zlib-devel``.
The following packages are optional:
-``cfitsio-devel``, ``gdal-devel``, ``geos-devel``, ``libcmocka-devel``,
-``libcurl-devel``, ``libxml2-devel``, ``netcdf-devel``, ``proj-devel``,
-``unixODBC-devel``.
+``cfitsio-devel``, ``gdal-devel``, ``geos-devel``, ``htslib-devel``,
+``libcmocka-devel``, ``libcurl-devel``, ``libxml2-devel``,
+``netcdf-devel``, ``proj-devel``, ``unixODBC-devel``.
On Ubuntu and Debian the following packages are required:
``bison``, ``cmake``, ``gcc``, ``pkg-config``, ``python3``,
@@ -116,8 +116,8 @@ The following packages are optional but
The following packages are optional:
``libcfitsio-dev``, ``libcmocka-dev``, ``libcurl4-gnutls-dev``,
-``libgdal-dev``, ``libgeos-dev``, ``libnetcdf-dev``, ``libproj-dev``,
-``libxml2-dev``, ``unixodbc-dev``.
+``libgdal-dev``, ``libgeos-dev``, ``libhts-dev``, ``libnetcdf-dev``,
+``libproj-dev``, ``libxml2-dev``, ``unixodbc-dev``.
``cmake`` must be at least version 3.12, ``python`` must be at least
version 3.5.
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -127,7 +127,7 @@ restrict_t BATgetaccess(BAT *b);
ValPtr BATgetprop(BAT *b, enum prop_t idx);
ValPtr BATgetprop_nolock(BAT *b, enum prop_t idx);
gdk_return BATgroup(BAT **groups, BAT **extents, BAT **histo, BAT *b, BAT *s,
BAT *g, BAT *e, BAT *h);
-const char *BATgroupaggrinit(BAT *b, BAT *g, BAT *e, BAT *s, oid *minp, oid
*maxp, BUN *ngrpp, struct canditer *ci);
+const char *BATgroupaggrinit(BAT *b, BAT *g, BAT *e, BAT *s, oid *minp, oid
*maxp, BUN *ngrpp, struct canditer *ci, BATiter *bi);
gdk_return BATgroupavg(BAT **bnp, BAT **cntsp, BAT *b, BAT *g, BAT *e, BAT *s,
int tp, bool skip_nils, int scale);
gdk_return BATgroupavg2(BAT **bnp, BAT **cntsp, BAT *b, BAT *g, BAT *e, BAT
*s, int tp, BUN ngrp, bool skip_nils, int scale);
gdk_return BATgroupavg3(BAT **avgp, BAT **remp, BAT **cntp, BAT *b, BAT *g,
BAT *e, BAT *s, bool skip_nils, bool inout);
@@ -473,6 +473,7 @@ ssize_t bteToStr(allocator *ma, char **d
const bte bte_nil;
oid canditer_idx(const struct canditer *ci, BUN p);
void canditer_init(struct canditer *ci, BAT *b, BAT *s);
+void canditer_init_bi(struct canditer *ci, BATiter *bi, BAT *s);
oid canditer_last(const struct canditer *ci);
oid canditer_mask_next(const struct canditer *ci, oid o, bool next);
oid canditer_peek(const struct canditer *ci);
diff --git a/cmake/Modules/FindHTSLIB.cmake b/cmake/Modules/FindHTSLIB.cmake
new file mode 100644
--- /dev/null
+++ b/cmake/Modules/FindHTSLIB.cmake
@@ -0,0 +1,47 @@
+# - Find htslib
+# Find the native htslib headers and libraries.
+#
+# HTSLIB_INCLUDE_DIRS - where to find hts.h, etc.
+# HTSLIB_LIBRARIES - List of libraries when using htslib.
+# HTSLIB_FOUND - True if htslib found.
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_HTSLIB QUIET htslib>=1.10)
+
+if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" OR PC_HTSLIB_FOUND)
+ if(NOT WIN32)
+
+ # Look for the header file.
+ find_path(HTSLIB_INCLUDE_DIR NAMES htslib/hts.h
+ HINTS
+ ${PC_HTSLIB_INCLUDEDIR}
+ ${PC_HTSLIB_INCLUDE_DIRS}
+ PATH_SUFFIXES htslib
+ )
+
+ # Look for the library.
+ find_library(HTSLIB_LIBRARIES NAMES hts
+ HINTS
+ ${PC_HTSLIB_LIBDIR}
+ ${PC_HTSLIB_LIBRARY_DIRS}
+ )
+
+ endif()
+endif()
+
+# Handle the QUIETLY and REQUIRED arguments and set HTSLIB_FOUND to TRUE if
all listed variables are TRUE.
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(HTSLIB DEFAULT_MSG HTSLIB_LIBRARIES
HTSLIB_INCLUDE_DIR)
+
+mark_as_advanced(HTSLIB_INCLUDE_DIR HTSLIB_LIBRARIES)
+
+if(HTSLIB_FOUND)
+ add_library(HTSLIB::HTSLIB UNKNOWN IMPORTED)
+ set_target_properties(HTSLIB::HTSLIB
+ PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${HTSLIB_INCLUDE_DIR}")
+ set_target_properties(HTSLIB::HTSLIB
+ PROPERTIES
+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${HTSLIB_LIBRARIES}")
+endif()
diff --git a/cmake/monetdb-defines.cmake b/cmake/monetdb-defines.cmake
--- a/cmake/monetdb-defines.cmake
+++ b/cmake/monetdb-defines.cmake
@@ -151,6 +151,7 @@ macro(monetdb_macro_variables)
set(SANITIZER ${SANITIZER})
set(HAVE_RTREE ${RTREE_FOUND})
set(HAVE_OPENSSL ${OPENSSL_FOUND})
+ set(HAVE_HTSLIB ${HTSLIB_FOUND})
set(BUILD_TYPE "${CMAKE_BUILD_TYPE}")
if(CMAKE_BUILD_TYPE)
diff --git a/cmake/monetdb-findpackages.cmake b/cmake/monetdb-findpackages.cmake
--- a/cmake/monetdb-findpackages.cmake
+++ b/cmake/monetdb-findpackages.cmake
@@ -138,6 +138,10 @@ if(WITH_OPENSSL)
find_package(OpenSSL 1.1.1)
endif()
+if(WITH_HTSLIB)
+ find_package(HTSLIB)
+endif()
+
find_package(Sphinx)
find_package(Semodule)
find_package(Awk)
diff --git a/cmake/monetdb-functions.cmake b/cmake/monetdb-functions.cmake
--- a/cmake/monetdb-functions.cmake
+++ b/cmake/monetdb-functions.cmake
@@ -81,6 +81,7 @@ function(monetdb_cmake_summary)
message(STATUS "Linux distro: ${LINUX_DISTRO}")
message(STATUS "Linux distro version: ${LINUX_DISTRO_VERSION}")
endif()
+ message(STATUS "hts library: ${HTSLIB_FOUND}")
message(STATUS "Iconv library: ${Iconv_FOUND}")
message(STATUS "Pthread library: ${CMAKE_USE_PTHREADS_INIT}")
message(STATUS "Pcre library: ${PCRE_FOUND}")
diff --git a/cmake/monetdb-options.cmake b/cmake/monetdb-options.cmake
--- a/cmake/monetdb-options.cmake
+++ b/cmake/monetdb-options.cmake
@@ -149,6 +149,10 @@ option(WITH_OPENSSL
"Include TLS support"
ON)
+option(WITH_HTSLIB
+ "Include htslib support"
+ ON)
+
option(CMAKE_SUMMARY
"Show a summary of the cmake configuration (for debug purposes)"
OFF)
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -2311,6 +2311,7 @@ typedef struct BATiter {
Heap *vh;
BUN count;
BUN baseoff;
+ oid hseq;
oid tseq;
BUN hfree, vhfree;
BUN nokey[2];
@@ -2331,11 +2332,12 @@ typedef struct BATiter {
copiedtodisk:1,
transient:1,
ascii:1;
- bat ustr;
restrict_t restricted:2;
#ifndef NDEBUG
bool locked:1;
#endif
+ bat id;
+ bat ustr;
union {
oid tvid;
bool tmsk;
@@ -2350,6 +2352,7 @@ bat_iterator_nolock(BAT *b)
const bool isview = VIEWtparent(b) != 0;
return (BATiter) {
.b = b,
+ .id = b->batCacheid,
.h = b->theap,
.base = b->theap->base
? b->theap->base + (b->tbaseoff << b->tshift)
@@ -2360,6 +2363,7 @@ bat_iterator_nolock(BAT *b)
.width = b->twidth,
.shift = b->tshift,
.type = b->ttype,
+ .hseq = b->hseqbase,
.tseq = b->tseqbase,
/* don't use b->theap->free in case b is a slice */
.hfree = b->ttype
@@ -2487,6 +2491,15 @@ bat_iterator_end(BATiter *bip)
*bip = (BATiter) {0};
}
+ATTR_PURE
+static inline bat
+VIEWtparentbi(const BATiter *bi)
+{
+ return bi->h == NULL || bi->h->parentid == bi->id
+ ? 0
+ : bi->h->parentid;
+}
+
gdk_export gdk_return HEAP_initialize(
Heap *heap, /* nbytes -- Initial size of the heap. */
size_t nbytes, /* alignment -- for objects on the heap. */
@@ -2746,6 +2759,8 @@ canditer_next(struct canditer *ci)
gdk_export void canditer_init(struct canditer *ci, BAT *b, BAT *s)
ATTR_ACCESS(write_only, 1);
+gdk_export void canditer_init_bi(struct canditer *ci, BATiter *bi, BAT *s)
+ ATTR_ACCESS(write_only, 1);
gdk_export oid canditer_peek(const struct canditer *ci)
ATTR_PURE;
gdk_export oid canditer_last(const struct canditer *ci)
@@ -2914,29 +2929,35 @@ BUNtail(BATiter *bi, BUN p)
* works with any TYPE_void or TYPE_oid bat */
ATTR_PURE
static inline oid
-BUNtoid(BAT *b, BUN p)
+BUNtoid(BATiter *bi, BUN p)
{
- assert(ATOMtype(b->ttype) == TYPE_oid);
+ assert(ATOMtype(bi->type) == TYPE_oid);
/* BATcount is the number of valid entries, so with
* exceptions, the last value can well be larger than
- * b->tseqbase + BATcount(b) */
- assert(p < BATcount(b));
- assert(b->ttype == TYPE_void || b->tvheap == NULL);
- if (is_oid_nil(b->tseqbase)) {
- if (b->ttype == TYPE_void)
+ * bi->seqbase + BATcount(b) */
+ assert(p < bi->count);
+ assert(bi->type == TYPE_void || bi->vh == NULL);
+ if (is_oid_nil(bi->tseq)) {
+ if (bi->type == TYPE_void)
return oid_nil;
- MT_lock_set(&b->theaplock);
- oid o = ((const oid *) b->theap->base)[p + b->tbaseoff];
- MT_lock_unset(&b->theaplock);
- return o;
+ return ((const oid *) bi->base)[p];
+ }
+ if (bi->type == TYPE_oid || bi->vh == NULL) {
+ return bi->tseq + p;
}
- if (b->ttype == TYPE_oid || b->tvheap == NULL) {
- return b->tseqbase + p;
- }
- /* b->tvheap != NULL, so we know there will be no parallel
- * modifications (so no locking) */
- BATiter bi = bat_iterator_nolock(b);
- return * (const oid *) BUNtpos(&bi, p);
+ /* bi->vh != NULL, so we know there will be no parallel
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
