Changeset: a3cd1224c973 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/a3cd1224c973
Branch: default
Log Message:
Merge samloader branch into default branch.
diffs (truncated from 2033 to 300 lines):
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -48,7 +48,7 @@ jobs:
- name: install packages
run: |
sudo apt-get update
- sudo apt-get install -y bison libbz2-dev libpcre2-dev liblzma-dev
liblz4-dev libxml2-dev pkg-config libxxhash-dev unixodbc-dev zlib1g-dev
libssl-dev
+ sudo apt-get install -y bison libbz2-dev libpcre2-dev liblzma-dev
liblz4-dev libxml2-dev pkg-config libxxhash-dev unixodbc-dev zlib1g-dev
libssl-dev libhts-dev
if: runner.os == 'Linux'
- name: Patch /etc/hosts on Unix
@@ -74,7 +74,7 @@ jobs:
if: runner.os == 'Linux'
- name: brew packages
- run: brew install bison pkgconf xxhash
+ run: brew install bison pkgconf xxhash htslib
if: runner.os == 'macOS'
- name: make MonetDB on macos
diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,21 @@
# ChangeLog file for devel
# This file is updated with Maddlog
+* Thu Oct 22 2026 Martin van Dinther <[email protected]>
+- Added support for reading DNA sequence alignment data from *.sam and *.bam
files.
+ Usage examples:
+ SELECT * FROM '/path/to/file1.bam';
+ or
+ SELECT * FROM file_loader('/path/to/file1.bam');
+ Prerequisites:
+ We use the HTSlib for reading these files and the library needs to be
+ pre-installed on your OS (e.g. on Fedora via: sudo dnf install htslib).
+ See also: https://www.htslib.org/download/
+ The HTSlib must be version 1.24 or higher.
+ Also the MonetDB server has to be started with startup argument:
+ --loadmodule htslib
+ to enable it. By default it is not enabled.
+
* Wed Oct 21 2026 Sjoerd Mullender <[email protected]>
- There is a new type of database farm, like the --dbextra farm, for
temporary heaps. Temporary heaps are heaps which are created and
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -102,9 +102,9 @@ The following packages are optional but
``readline-devel``, ``xz-devel``, ``zlib-devel``.
The following packages are optional:
-``cfitsio-devel``, ``gdal-devel``, ``geos-devel``, ``libcmocka-devel``,
-``libcurl-devel``, ``libxml2-devel``, ``netcdf-devel``, ``proj-devel``,
-``unixODBC-devel``.
+``cfitsio-devel``, ``gdal-devel``, ``geos-devel``, ``htslib-devel``,
+``libcmocka-devel``, ``libcurl-devel``, ``libxml2-devel``,
+``netcdf-devel``, ``proj-devel``, ``unixODBC-devel``.
On Ubuntu and Debian the following packages are required:
``bison``, ``cmake``, ``gcc``, ``pkg-config``, ``python3``,
@@ -116,8 +116,8 @@ The following packages are optional but
The following packages are optional:
``libcfitsio-dev``, ``libcmocka-dev``, ``libcurl4-gnutls-dev``,
-``libgdal-dev``, ``libgeos-dev``, ``libnetcdf-dev``, ``libproj-dev``,
-``libxml2-dev``, ``unixodbc-dev``.
+``libgdal-dev``, ``libgeos-dev``, ``libhts-dev``, ``libnetcdf-dev``,
+``libproj-dev``, ``libxml2-dev``, ``unixodbc-dev``.
``cmake`` must be at least version 3.12, ``python`` must be at least
version 3.5.
diff --git a/cmake/Modules/FindHTSLIB.cmake b/cmake/Modules/FindHTSLIB.cmake
new file mode 100644
--- /dev/null
+++ b/cmake/Modules/FindHTSLIB.cmake
@@ -0,0 +1,47 @@
+# - Find htslib
+# Find the native htslib headers and libraries.
+#
+# HTSLIB_INCLUDE_DIRS - where to find hts.h, etc.
+# HTSLIB_LIBRARIES - List of libraries when using htslib.
+# HTSLIB_FOUND - True if htslib found.
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_HTSLIB QUIET htslib>=1.10)
+
+if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" OR PC_HTSLIB_FOUND)
+ if(NOT WIN32)
+
+ # Look for the header file.
+ find_path(HTSLIB_INCLUDE_DIR NAMES htslib/hts.h
+ HINTS
+ ${PC_HTSLIB_INCLUDEDIR}
+ ${PC_HTSLIB_INCLUDE_DIRS}
+ PATH_SUFFIXES htslib
+ )
+
+ # Look for the library.
+ find_library(HTSLIB_LIBRARIES NAMES hts
+ HINTS
+ ${PC_HTSLIB_LIBDIR}
+ ${PC_HTSLIB_LIBRARY_DIRS}
+ )
+
+ endif()
+endif()
+
+# Handle the QUIETLY and REQUIRED arguments and set HTSLIB_FOUND to TRUE if
all listed variables are TRUE.
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(HTSLIB DEFAULT_MSG HTSLIB_LIBRARIES
HTSLIB_INCLUDE_DIR)
+
+mark_as_advanced(HTSLIB_INCLUDE_DIR HTSLIB_LIBRARIES)
+
+if(HTSLIB_FOUND)
+ add_library(HTSLIB::HTSLIB UNKNOWN IMPORTED)
+ set_target_properties(HTSLIB::HTSLIB
+ PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${HTSLIB_INCLUDE_DIR}")
+ set_target_properties(HTSLIB::HTSLIB
+ PROPERTIES
+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${HTSLIB_LIBRARIES}")
+endif()
diff --git a/cmake/monetdb-defines.cmake b/cmake/monetdb-defines.cmake
--- a/cmake/monetdb-defines.cmake
+++ b/cmake/monetdb-defines.cmake
@@ -151,6 +151,7 @@ macro(monetdb_macro_variables)
set(SANITIZER ${SANITIZER})
set(HAVE_RTREE ${RTREE_FOUND})
set(HAVE_OPENSSL ${OPENSSL_FOUND})
+ set(HAVE_HTSLIB ${HTSLIB_FOUND})
set(BUILD_TYPE "${CMAKE_BUILD_TYPE}")
if(CMAKE_BUILD_TYPE)
diff --git a/cmake/monetdb-findpackages.cmake b/cmake/monetdb-findpackages.cmake
--- a/cmake/monetdb-findpackages.cmake
+++ b/cmake/monetdb-findpackages.cmake
@@ -138,6 +138,10 @@ if(WITH_OPENSSL)
find_package(OpenSSL 1.1.1)
endif()
+if(WITH_HTSLIB)
+ find_package(HTSLIB)
+endif()
+
find_package(Sphinx)
find_package(Semodule)
find_package(Awk)
diff --git a/cmake/monetdb-functions.cmake b/cmake/monetdb-functions.cmake
--- a/cmake/monetdb-functions.cmake
+++ b/cmake/monetdb-functions.cmake
@@ -81,6 +81,7 @@ function(monetdb_cmake_summary)
message(STATUS "Linux distro: ${LINUX_DISTRO}")
message(STATUS "Linux distro version: ${LINUX_DISTRO_VERSION}")
endif()
+ message(STATUS "hts library: ${HTSLIB_FOUND}")
message(STATUS "Iconv library: ${Iconv_FOUND}")
message(STATUS "Pthread library: ${CMAKE_USE_PTHREADS_INIT}")
message(STATUS "Pcre library: ${PCRE_FOUND}")
diff --git a/cmake/monetdb-options.cmake b/cmake/monetdb-options.cmake
--- a/cmake/monetdb-options.cmake
+++ b/cmake/monetdb-options.cmake
@@ -149,6 +149,10 @@ option(WITH_OPENSSL
"Include TLS support"
ON)
+option(WITH_HTSLIB
+ "Include htslib support"
+ ON)
+
option(CMAKE_SUMMARY
"Show a summary of the cmake configuration (for debug purposes)"
OFF)
diff --git a/monetdb_config.h.in b/monetdb_config.h.in
--- a/monetdb_config.h.in
+++ b/monetdb_config.h.in
@@ -369,6 +369,7 @@
#cmakedefine HAVE_LIBZ 1
#cmakedefine HAVE_NETCDF 1
#cmakedefine HAVE_OPENSSL 1
+#cmakedefine HAVE_HTSLIB 1
#cmakedefine HAVE_PROJ 1
#cmakedefine HAVE_READLINE 1
#cmakedefine HAVE_RTREE 1
diff --git a/sql/backends/monet5/vaults/CMakeLists.txt
b/sql/backends/monet5/vaults/CMakeLists.txt
--- a/sql/backends/monet5/vaults/CMakeLists.txt
+++ b/sql/backends/monet5/vaults/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(netcdf)
add_subdirectory(shp)
add_subdirectory(csv)
add_subdirectory(parquet)
+add_subdirectory(sambam)
add_subdirectory(monetdb)
add_subdirectory(odbc)
diff --git a/sql/backends/monet5/vaults/sambam/CMakeLists.txt
b/sql/backends/monet5/vaults/sambam/CMakeLists.txt
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/vaults/sambam/CMakeLists.txt
@@ -0,0 +1,60 @@
+#[[
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# For copyright information, see the file debian/copyright.
+#]]
+
+if(HAVE_HTSLIB)
+ add_library(htslib MODULE)
+
+ target_sources(htslib
+ PRIVATE
+ sambam.c)
+
+ target_include_directories(htslib
+ PRIVATE
+ $<$<BOOL:${HTSLIB_FOUND}>:${HTSLIB_INCLUDE_DIRS}>
+ $<TARGET_PROPERTY:mal,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:malmodules,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:atoms,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sql,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlcommon,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlserver,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:sqlstorage,INTERFACE_INCLUDE_DIRECTORIES>
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+ $<INSTALL_INTERFACE:${INCLUDEDIR}/monetdb>)
+
+ set_target_properties(htslib
+ PROPERTIES
+ NO_SONAME ON
+ OUTPUT_NAME
+ _htslib)
+
+ target_compile_definitions(htslib
+ PRIVATE
+ LIBHTSLIB)
+
+ target_link_libraries(htslib
+ PRIVATE
+ monetdb_config_header
+ bat
+ monetdb5
+ sqlinclude
+ HTSLIB::HTSLIB)
+
+ install(TARGETS htslib
+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/monetdb5-${MONETDB_VERSION}
+ COMPONENT htslib)
+
+ if(WIN32)
+ install(FILES
+ $<TARGET_PDB_FILE:htslib>
+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/monetdb5-${MONETDB_VERSION}
+ OPTIONAL)
+ endif()
+endif()
+
diff --git a/sql/backends/monet5/vaults/sambam/sambam.c
b/sql/backends/monet5/vaults/sambam/sambam.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/vaults/sambam/sambam.c
@@ -0,0 +1,554 @@
+/*
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * For copyright information, see the file debian/copyright.
+ */
+
+/*
+ * This file contains a reader of aligmnent data from *.sam and *.bam files
+ * and load the alignment data into a transient MonetDB table consisting of 12
columns.
+ * SAM stands for Sequence Alignment Map. Ref:
https://en.wikipedia.org/wiki/SAM_(file_format)
+ * BAM stands for Binary Alignment Map. Ref:
https://en.wikipedia.org/wiki/BAM_(file_format)
+ * BAM is the compressed binary representation of SAM (Sequence Alignment Map).
+ * BAM is one of the most common file formats for raw genome sequencing data.
+ *
+ * A SAM file is a TAB-delimited text format consisting of a header section,
which is optional,
+ * and an alignment section. If present, the header must be prior to the
alignments.
+ * Header lines start with ‘@’, while alignment lines do not.
+ * Each alignment line has 11 mandatory fields for essential alignment
information
+ * such as mapping position, and variable number of optional fields for
flexible
+ * or aligner specific information.
+ * Ref: https://samtools.github.io/hts-specs/SAMv1.pdf
+ *
+ * We use the htslib library for reading data from these files.
+ * Ref: https://github.com/samtools/htslib and https://www.htslib.org/
+ * To install this library and its include files on Fedora, do: sudo dnf
install htslib-devel
+ *
+ * Limitations:
+ * - Only the alignment data is read and converted into bats, not the
+ * header information as it has a different table format.
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
