This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new fac2856a feat(r): Add LZ4 compression support for IPC read (#916)
fac2856a is described below
commit fac2856ac56e950c4da60bf2732ce6a277d8de61
Author: Dewey Dunnington <[email protected]>
AuthorDate: Thu Jul 30 21:58:18 2026 -0400
feat(r): Add LZ4 compression support for IPC read (#916)
This PR adds LZ4 to the configuration script for R to pick it up if it's
provided by the environment.
``` r
library(nanoarrow)
library(reticulate)
# IPC Write with compression not available in arrow/R
pa <- reticulate::import("pyarrow")
io <- reticulate::import("io")
buf <- io$BytesIO()
batch <- arrow::record_batch(x = 1:1000)
writer <- pa$ipc$new_stream(buf, batch$schema, options =
pa$ipc$IpcWriteOptions(compression = "lz4"))
writer$write_batch(batch)
writer$close()
nanoarrow::read_nanoarrow(as.raw(buf$getvalue())) |>
tibble::as_tibble()
#> # A tibble: 1,000 × 1
#> x
#> <int>
#> 1 1
#> 2 2
#> 3 3
#> 4 4
#> 5 5
#> 6 6
#> 7 7
#> 8 8
#> 9 9
#> 10 10
#> # ℹ 990 more rows
```
<sup>Created on 2026-07-30 with [reprex
v2.1.1](https://reprex.tidyverse.org)</sup>
---
r/DESCRIPTION | 2 +-
r/NAMESPACE | 1 +
r/R/convert-array.R | 4 ++--
r/R/ipc.R | 24 +++++++++++++++++++++---
r/R/nanoarrow-package.R | 7 +++++++
r/configure | 21 +++++++++++++++++++++
r/man/convert_array.Rd | 4 ++--
r/man/example_ipc_stream.Rd | 4 ++--
r/man/nanoarrow_version.Rd | 4 ++++
r/src/Makevars.ucrt | 18 +++++++++++++++---
r/src/Makevars.win | 18 ++++++++++++++++--
r/src/init.c | 2 ++
r/src/version.c | 8 ++++++++
r/tests/testthat/test-ipc.R | 13 ++++++++++++-
r/{src/version.c => tools/test_lz4.c} | 24 +++++++++---------------
15 files changed, 123 insertions(+), 31 deletions(-)
diff --git a/r/DESCRIPTION b/r/DESCRIPTION
index 60e4522f..dceb0b4b 100644
--- a/r/DESCRIPTION
+++ b/r/DESCRIPTION
@@ -34,7 +34,7 @@ Suggests:
tibble,
vctrs,
withr
-SystemRequirements: libzstd (optional)
+SystemRequirements: libzstd (optional), liblz4 (optional)
Config/testthat/edition: 3
Config/build/bootstrap: TRUE
Config/roxygen2/version: 8.0.0
diff --git a/r/NAMESPACE b/r/NAMESPACE
index a4b4df14..cb27c9ae 100644
--- a/r/NAMESPACE
+++ b/r/NAMESPACE
@@ -218,6 +218,7 @@ export(nanoarrow_schema_modify)
export(nanoarrow_schema_parse)
export(nanoarrow_vctr)
export(nanoarrow_version)
+export(nanoarrow_with_lz4)
export(nanoarrow_with_zstd)
export(read_nanoarrow)
export(register_nanoarrow_extension)
diff --git a/r/R/convert-array.R b/r/R/convert-array.R
index ba9f3a51..3333a23e 100644
--- a/r/R/convert-array.R
+++ b/r/R/convert-array.R
@@ -53,7 +53,7 @@
#' - [character()]: String and large string types can be converted to
#' [character()]. The conversion does not check for valid UTF-8: if you need
#' finer-grained control over encodings, use `to = blob::blob()`.
-#' - [factor()]: Dictionary-encoded arrays of strings can be converted to
+#' - [base::factor()]: Dictionary-encoded arrays of strings can be converted to
#' `factor()`; however, this must be specified explicitly (i.e.,
#' `convert_array(array, factor())`) because arrays arriving
#' in chunks can have dictionaries that contain different levels. Use
@@ -68,7 +68,7 @@
#' be converted to [blob::blob()].
#' - [vctrs::list_of()]: List, large list, and fixed-size list types can be
#' converted to [vctrs::list_of()].
-#' - [matrix()]: Fixed-size list types can be converted to
+#' - [base::matrix()]: Fixed-size list types can be converted to
#' `matrix(ptype, ncol = fixed_size)`.
#' - [data.frame()]: Struct types can be converted to [data.frame()].
#' - [vctrs::unspecified()]: Any type can be converted to
[vctrs::unspecified()];
diff --git a/r/R/ipc.R b/r/R/ipc.R
index 881b0f36..9b2e53fa 100644
--- a/r/R/ipc.R
+++ b/r/R/ipc.R
@@ -149,17 +149,17 @@ write_nanoarrow.character <- function(data, x, ...) {
#'
#' An example stream that can be used for testing or examples.
#'
-#' @param compression One of "none" or "zstd"
+#' @param compression One of "none", "zstd", or "lz4"
#'
#' @return A raw vector that can be passed to [read_nanoarrow()]
#' @export
#'
#' @examples
#' as.data.frame(read_nanoarrow(example_ipc_stream()))
-example_ipc_stream <- function(compression = c("none", "zstd")) {
+example_ipc_stream <- function(compression = c("none", "zstd", "lz4")) {
compression <- match.arg(compression)
- # data.frame(some_col = c(1L, 2L, 3L)) as a serialized schema/batch
+ # data.frame(some_col = c(0L, 1L, 2L)) as a serialized schema/batch
schema <- as.raw(c(
0xff, 0xff, 0xff, 0xff, 0x10, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,
0x00, 0x00,
0x0a, 0x00, 0x0e, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00,
0x00, 0x00,
@@ -201,6 +201,24 @@ example_ipc_stream <- function(compression = c("none",
"zstd")) {
0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02,
0x00, 0x00,
0x00, 0x00, 0x00, 0x00
))
+ } else if (identical(compression, "lz4")) {
+ batch <- as.raw(c(
+ 0xff, 0xff, 0xff, 0xff, 0x98, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x0c, 0x00, 0x18, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00,
0x0c, 0x00,
+ 0x0c, 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x1c, 0x00, 0x00, 0x00,
0x28, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00,
0x1c, 0x00,
+ 0x10, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x0c, 0x00, 0x00, 0x00,
0x48, 0x00,
+ 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
0x04, 0x00,
+ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x04, 0x22, 0x4d, 0x18, 0x60, 0x40, 0x82, 0x0c, 0x00, 0x00, 0x80, 0x00,
0x00, 0x00,
+ 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00
+ ))
} else {
batch <- as.raw(c(
0xff, 0xff, 0xff, 0xff, 0x88, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
0x00, 0x00,
diff --git a/r/R/nanoarrow-package.R b/r/R/nanoarrow-package.R
index e2bd3c60..7d585384 100644
--- a/r/R/nanoarrow-package.R
+++ b/r/R/nanoarrow-package.R
@@ -36,6 +36,7 @@ NULL
#' @examples
#' nanoarrow_version()
#' nanoarrow_with_zstd()
+#' nanoarrow_with_lz4()
nanoarrow_version <- function(runtime = TRUE) {
if (runtime) {
.Call(nanoarrow_c_version_runtime)
@@ -49,3 +50,9 @@ nanoarrow_version <- function(runtime = TRUE) {
nanoarrow_with_zstd <- function() {
.Call(nanoarrow_c_with_zstd)
}
+
+#' @rdname nanoarrow_version
+#' @export
+nanoarrow_with_lz4 <- function() {
+ .Call(nanoarrow_c_with_lz4)
+}
diff --git a/r/configure b/r/configure
index 301bb4d1..42c8385c 100755
--- a/r/configure
+++ b/r/configure
@@ -50,6 +50,27 @@ fi
rm -f tools/test_zstd.o test_zstd test_zstd.log || true
+# Check to see if a lz4 test file compiles and links without any help from
pkg-config
+LZ4_FOUND=""
+PKG_CPPFLAGS="$PKG_CPPFLAGS" PKG_LIBS="$PKG_LIBS -llz4" \
+ $R_HOME/bin/R CMD SHLIB tools/test_lz4.c -o test_lz4 >test_lz4.log 2>&1
+if [ $? -eq 0 ]; then
+ echo "tools/test_lz4.c compiled without error"
+ PKG_CPPFLAGS="$PKG_CPPFLAGS -DNANOARROW_IPC_WITH_LZ4"
+ PKG_LIBS="$PKG_LIBS -llz4"
+ LZ4_FOUND="yes"
+fi
+
+# Add pkg-config for liblz4 if possible
+if [ -z "$LZ4_FOUND" ] && pkg-config liblz4 --exists >/dev/null 2>&1; then
+ PKG_CPPFLAGS="`pkg-config liblz4 --cflags` -DNANOARROW_IPC_WITH_LZ4
$PKG_CPPFLAGS"
+ PKG_LIBS="`pkg-config liblz4 --libs` $PKG_LIBS"
+ echo "Using pkg-config liblz4"
+ LZ4_FOUND="yes"
+fi
+
+rm -f tools/test_lz4.o test_lz4 test_lz4.log || true
+
echo "Using PKG_CPPFLAGS=$PKG_CPPFLAGS"
echo "Using PKG_LIBS=$PKG_LIBS"
diff --git a/r/man/convert_array.Rd b/r/man/convert_array.Rd
index 79377e94..dc43c17f 100644
--- a/r/man/convert_array.Rd
+++ b/r/man/convert_array.Rd
@@ -45,7 +45,7 @@ through a floating-point double (e.g., very large uint64 and
int64 values).
\item \code{\link[=character]{character()}}: String and large string types can
be converted to
\code{\link[=character]{character()}}. The conversion does not check for valid
UTF-8: if you need
finer-grained control over encodings, use \code{to = blob::blob()}.
-\item \code{\link[=factor]{factor()}}: Dictionary-encoded arrays of strings
can be converted to
+\item \code{\link[base:factor]{base::factor()}}: Dictionary-encoded arrays of
strings can be converted to
\code{factor()}; however, this must be specified explicitly (i.e.,
\code{convert_array(array, factor())}) because arrays arriving
in chunks can have dictionaries that contain different levels. Use
@@ -60,7 +60,7 @@ attribute of \code{to}.
be converted to \code{\link[blob:blob]{blob::blob()}}.
\item \code{\link[vctrs:list_of]{vctrs::list_of()}}: List, large list, and
fixed-size list types can be
converted to \code{\link[vctrs:list_of]{vctrs::list_of()}}.
-\item \code{\link[=matrix]{matrix()}}: Fixed-size list types can be converted
to
+\item \code{\link[base:matrix]{base::matrix()}}: Fixed-size list types can be
converted to
\code{matrix(ptype, ncol = fixed_size)}.
\item \code{\link[=data.frame]{data.frame()}}: Struct types can be converted
to \code{\link[=data.frame]{data.frame()}}.
\item \code{\link[vctrs:unspecified]{vctrs::unspecified()}}: Any type can be
converted to \code{\link[vctrs:unspecified]{vctrs::unspecified()}};
diff --git a/r/man/example_ipc_stream.Rd b/r/man/example_ipc_stream.Rd
index 14ad124c..410a0117 100644
--- a/r/man/example_ipc_stream.Rd
+++ b/r/man/example_ipc_stream.Rd
@@ -4,10 +4,10 @@
\alias{example_ipc_stream}
\title{Example Arrow IPC Data}
\usage{
-example_ipc_stream(compression = c("none", "zstd"))
+example_ipc_stream(compression = c("none", "zstd", "lz4"))
}
\arguments{
-\item{compression}{One of "none" or "zstd"}
+\item{compression}{One of "none", "zstd", or "lz4"}
}
\value{
A raw vector that can be passed to
\code{\link[=read_nanoarrow]{read_nanoarrow()}}
diff --git a/r/man/nanoarrow_version.Rd b/r/man/nanoarrow_version.Rd
index 00fd3097..666f392b 100644
--- a/r/man/nanoarrow_version.Rd
+++ b/r/man/nanoarrow_version.Rd
@@ -3,11 +3,14 @@
\name{nanoarrow_version}
\alias{nanoarrow_version}
\alias{nanoarrow_with_zstd}
+\alias{nanoarrow_with_lz4}
\title{Underlying 'nanoarrow' C library build}
\usage{
nanoarrow_version(runtime = TRUE)
nanoarrow_with_zstd()
+
+nanoarrow_with_lz4()
}
\arguments{
\item{runtime}{Compare TRUE and FALSE values to detect a
@@ -23,4 +26,5 @@ Underlying 'nanoarrow' C library build
\examples{
nanoarrow_version()
nanoarrow_with_zstd()
+nanoarrow_with_lz4()
}
diff --git a/r/src/Makevars.ucrt b/r/src/Makevars.ucrt
index e9abf967..be15e8e8 100644
--- a/r/src/Makevars.ucrt
+++ b/r/src/Makevars.ucrt
@@ -15,8 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-# This Makevars handles R >= 4.2 on Windows (pkg-config is available on all
such versions)
+# This Makevars handles R >= 4.2 on Windows
+# Note: configure.win is excluded from source tarballs, so this file is used
directly
+# on CRAN. We only enable compression if pkg-config finds the libraries.
+# Check for zstd via pkg-config
ifeq (,$(shell pkg-config libzstd --libs 2>/dev/null))
ZSTD_CFLAGS =
ZSTD_LIB_FLAGS =
@@ -25,5 +28,14 @@ else
ZSTD_LIB_FLAGS = $(shell pkg-config --libs libzstd)
endif
-PKG_CPPFLAGS = -I../inst/include -I../src -DFLATCC_USE_GENERIC_ALIGNED_ALLOC
$(ZSTD_CFLAGS)
-PKG_LIBS = $(ZSTD_LIB_FLAGS)
+# Check for lz4 via pkg-config (available in rtools43+)
+ifeq (,$(shell pkg-config liblz4 --libs 2>/dev/null))
+ LZ4_CFLAGS =
+ LZ4_LIB_FLAGS =
+else
+ LZ4_CFLAGS = $(shell pkg-config --cflags liblz4) -DNANOARROW_IPC_WITH_LZ4
+ LZ4_LIB_FLAGS = $(shell pkg-config --libs liblz4)
+endif
+
+PKG_CPPFLAGS = -I../inst/include -I../src -DFLATCC_USE_GENERIC_ALIGNED_ALLOC
$(ZSTD_CFLAGS) $(LZ4_CFLAGS)
+PKG_LIBS = $(ZSTD_LIB_FLAGS) $(LZ4_LIB_FLAGS)
diff --git a/r/src/Makevars.win b/r/src/Makevars.win
index 7efd4791..04388f70 100644
--- a/r/src/Makevars.win
+++ b/r/src/Makevars.win
@@ -32,5 +32,19 @@ else
endif
endif
-PKG_CPPFLAGS = -I../inst/include -I../src -DFLATCC_USE_GENERIC_ALIGNED_ALLOC
$(ZSTD_CFLAGS)
-PKG_LIBS = $(ZSTD_LIB_FLAGS)
+# Check for lz4 via pkg-config (64-bit only, same as zstd)
+ifeq (,$(shell pkg-config liblz4 --libs 2>/dev/null))
+ LZ4_CFLAGS =
+ LZ4_LIB_FLAGS =
+else
+ ifeq "$(WIN)" "64"
+ LZ4_CFLAGS = $(shell pkg-config --cflags liblz4) -DNANOARROW_IPC_WITH_LZ4
+ LZ4_LIB_FLAGS = $(shell pkg-config --libs liblz4)
+ else
+ LZ4_CFLAGS =
+ LZ4_LIB_FLAGS =
+ endif
+endif
+
+PKG_CPPFLAGS = -I../inst/include -I../src -DFLATCC_USE_GENERIC_ALIGNED_ALLOC
$(ZSTD_CFLAGS) $(LZ4_CFLAGS)
+PKG_LIBS = $(ZSTD_LIB_FLAGS) $(LZ4_LIB_FLAGS)
diff --git a/r/src/init.c b/r/src/init.c
index 70eec867..97f8f4c9 100644
--- a/r/src/init.c
+++ b/r/src/init.c
@@ -101,6 +101,7 @@ extern SEXP nanoarrow_c_vctr_as_slice(SEXP indices_sexp);
extern SEXP nanoarrow_c_version(void);
extern SEXP nanoarrow_c_version_runtime(void);
extern SEXP nanoarrow_c_with_zstd(void);
+extern SEXP nanoarrow_c_with_lz4(void);
static const R_CallMethodDef CallEntries[] = {
{"nanoarrow_c_make_altrep_chr", (DL_FUNC)&nanoarrow_c_make_altrep_chr, 1},
@@ -184,6 +185,7 @@ static const R_CallMethodDef CallEntries[] = {
{"nanoarrow_c_version", (DL_FUNC)&nanoarrow_c_version, 0},
{"nanoarrow_c_version_runtime", (DL_FUNC)&nanoarrow_c_version_runtime, 0},
{"nanoarrow_c_with_zstd", (DL_FUNC)&nanoarrow_c_with_zstd, 0},
+ {"nanoarrow_c_with_lz4", (DL_FUNC)&nanoarrow_c_with_lz4, 0},
{NULL, NULL, 0}};
/* end generated by tools/make-callentries.R */
diff --git a/r/src/version.c b/r/src/version.c
index 36095770..407e52b0 100644
--- a/r/src/version.c
+++ b/r/src/version.c
@@ -32,3 +32,11 @@ SEXP nanoarrow_c_with_zstd(void) {
return Rf_ScalarLogical(0);
#endif
}
+
+SEXP nanoarrow_c_with_lz4(void) {
+#if defined(NANOARROW_IPC_WITH_LZ4)
+ return Rf_ScalarLogical(1);
+#else
+ return Rf_ScalarLogical(0);
+#endif
+}
diff --git a/r/tests/testthat/test-ipc.R b/r/tests/testthat/test-ipc.R
index 67043e92..69f96aa2 100644
--- a/r/tests/testthat/test-ipc.R
+++ b/r/tests/testthat/test-ipc.R
@@ -123,7 +123,7 @@ test_that("read_nanoarrow() works for URLs", {
)
})
-test_that("read_nanoarrow() works with buffer compression", {
+test_that("read_nanoarrow() works with zstd buffer compression", {
skip_if_not(nanoarrow_with_zstd())
stream <- read_nanoarrow(example_ipc_stream(compression = "zstd"))
@@ -134,6 +134,17 @@ test_that("read_nanoarrow() works with buffer
compression", {
)
})
+test_that("read_nanoarrow() works with lz4 buffer compression", {
+ skip_if_not(nanoarrow_with_lz4())
+
+ stream <- read_nanoarrow(example_ipc_stream(compression = "lz4"))
+ expect_s3_class(stream, "nanoarrow_array_stream")
+ expect_identical(
+ as.data.frame(stream),
+ data.frame(some_col = c(0L, 1L, 2L))
+ )
+})
+
test_that("write_nanoarrow() works for URLs", {
tf <- tempfile()
on.exit(unlink(tf))
diff --git a/r/src/version.c b/r/tools/test_lz4.c
similarity index 67%
copy from r/src/version.c
copy to r/tools/test_lz4.c
index 36095770..bdb79acb 100644
--- a/r/src/version.c
+++ b/r/tools/test_lz4.c
@@ -15,20 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-#define R_NO_REMAP
-#include <R.h>
-#include <Rinternals.h>
+#include <lz4.h>
+#include <stdint.h>
+#include <string.h>
-#include "nanoarrow.h"
-
-SEXP nanoarrow_c_version(void) { return Rf_mkString(NANOARROW_VERSION); }
-
-SEXP nanoarrow_c_version_runtime(void) { return
Rf_mkString(ArrowNanoarrowVersion()); }
-
-SEXP nanoarrow_c_with_zstd(void) {
-#if defined(NANOARROW_IPC_WITH_ZSTD)
- return Rf_ScalarLogical(1);
-#else
- return Rf_ScalarLogical(0);
-#endif
+// Function that requires at least one symbol from lz4.h
+int test_lz4(void) {
+ uint8_t src[128];
+ memset(src, 0, sizeof(src));
+ uint8_t dst[128];
+ return LZ4_compress_default((const char*)src, (char*)dst, sizeof(src),
sizeof(dst));
}
