This is an automated email from the ASF dual-hosted git repository.
npr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 0f30b70 ARROW-7092: [R] Add vignette for dplyr and datasets
0f30b70 is described below
commit 0f30b70e873785042cd27d029fcd6b7a75e54821
Author: Neal Richardson <[email protected]>
AuthorDate: Fri Jan 17 19:45:27 2020 -0800
ARROW-7092: [R] Add vignette for dplyr and datasets
The vignette could probably use some polishing, and it's not awesome that
it's not a dynamic document (all of the code chunks and output are static,
pregenerated) but I'm not sure of a better way since we can't include the
example data in the package.
Feedback greatly appreciated. Given the push for the 0.16 release, I'll
probably merge this as soon as CI is green because I have another issue
(https://issues.apache.org/jira/browse/ARROW-7581) for documentation polishing
for the release and I don't want this to block that. I can incorporate
suggestions and edits folks have in that followup PR.
Closes #6222 from nealrichardson/dataset-vignette and squashes the
following commits:
cca95f053 <Neal Richardson> Add notes on extension points for datasets
e3ba51daf <Neal Richardson> First pass at (static) dataset vignette; also
add some print methods and various schema methods
Authored-by: Neal Richardson <[email protected]>
Signed-off-by: Neal Richardson <[email protected]>
---
r/NAMESPACE | 5 +-
r/R/arrowExports.R | 20 +++-
r/R/dataset.R | 21 ++--
r/R/dplyr.R | 20 ++++
r/R/schema.R | 70 +++++------
r/README.md | 135 +++++++++++++++------
r/man/Partitioning.Rd | 8 +-
r/man/Schema.Rd | 4 +-
r/man/hive_partition.Rd | 7 +-
r/man/read_schema.Rd | 5 +-
r/src/arrowExports.cpp | 77 +++++++++++-
r/src/schema.cpp | 31 ++++-
r/tests/testthat/test-dataset.R | 52 ++++++++
r/tests/testthat/test-schema.R | 14 +++
r/vignettes/dataset.Rmd | 260 ++++++++++++++++++++++++++++++++++++++++
15 files changed, 621 insertions(+), 108 deletions(-)
diff --git a/r/NAMESPACE b/r/NAMESPACE
index 2c38512..c0f7770 100644
--- a/r/NAMESPACE
+++ b/r/NAMESPACE
@@ -39,6 +39,7 @@ S3method(names,Schema)
S3method(names,Table)
S3method(print,"arrow-enum")
S3method(print,array_expression)
+S3method(print,arrow_dplyr_query)
S3method(read_message,InputStream)
S3method(read_message,MessageReader)
S3method(read_message,default)
@@ -46,10 +47,6 @@ S3method(read_record_batch,Buffer)
S3method(read_record_batch,InputStream)
S3method(read_record_batch,Message)
S3method(read_record_batch,raw)
-S3method(read_schema,Buffer)
-S3method(read_schema,InputStream)
-S3method(read_schema,Message)
-S3method(read_schema,raw)
S3method(read_table,RecordBatchFileReader)
S3method(read_table,RecordBatchStreamReader)
S3method(read_table,character)
diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R
index cfe3e47..8e608f1 100644
--- a/r/R/arrowExports.R
+++ b/r/R/arrowExports.R
@@ -1344,8 +1344,24 @@ Schema__field <- function(s, i){
.Call(`_arrow_Schema__field` , s, i)
}
-Schema__names <- function(schema){
- .Call(`_arrow_Schema__names` , schema)
+Schema__GetFieldByName <- function(s, x){
+ .Call(`_arrow_Schema__GetFieldByName` , s, x)
+}
+
+Schema__fields <- function(schema){
+ .Call(`_arrow_Schema__fields` , schema)
+}
+
+Schema__field_names <- function(schema){
+ .Call(`_arrow_Schema__field_names` , schema)
+}
+
+Schema__HasMetadata <- function(schema){
+ .Call(`_arrow_Schema__HasMetadata` , schema)
+}
+
+Schema__metadata <- function(schema){
+ .Call(`_arrow_Schema__metadata` , schema)
}
Schema__serialize <- function(schema){
diff --git a/r/R/dataset.R b/r/R/dataset.R
index 71a7468..5ad53c7 100644
--- a/r/R/dataset.R
+++ b/r/R/dataset.R
@@ -77,12 +77,14 @@ Dataset <- R6Class("Dataset", inherit = Object,
#' @description
#' Start a new scan of the data
#' @return A [ScannerBuilder]
- NewScan = function() unique_ptr(ScannerBuilder,
dataset___Dataset__NewScan(self))
+ NewScan = function() unique_ptr(ScannerBuilder,
dataset___Dataset__NewScan(self)),
+ ToString = function() self$schema$ToString()
),
active = list(
#' @description
#' Return the Dataset's `Schema`
- schema = function() shared_ptr(Schema, dataset___Dataset__schema(self))
+ schema = function() shared_ptr(Schema, dataset___Dataset__schema(self)),
+ metadata = function() self$schema$metadata
)
)
Dataset$create <- function(sources, schema) {
@@ -331,19 +333,19 @@ ScannerBuilder <- R6Class("ScannerBuilder", inherit =
Object,
#' @export
names.ScannerBuilder <- function(x) names(x$schema)
-#' Define a partitioning for a Source
+#' Define Partitioning for a Source
#'
#' @description
-#' Pass a `Partitioning` to a [FileSystemSourceFactory]'s `$create()`
+#' Pass a `Partitioning` object to a [FileSystemSourceFactory]'s `$create()`
#' method to indicate how the file's paths should be interpreted to define
#' partitioning.
#'
-#' A `DirectoryPartitioning` describes how to interpret raw path segments, in
+#' `DirectoryPartitioning` describes how to interpret raw path segments, in
#' order. For example, `schema(year = int16(), month = int8())` would define
#' partitions for file paths like "2019/01/file.parquet",
#' "2019/02/file.parquet", etc.
#'
-#' A `HivePartitioning` is for Hive-style partitioning, which embeds field
+#' `HivePartitioning` is for Hive-style partitioning, which embeds field
#' names and values in path segments, such as
#' "/year=2019/month=2/data.parquet". Because fields are named in the path
#' segments, order does not matter.
@@ -374,16 +376,15 @@ HivePartitioning$create <- function(schema) {
shared_ptr(HivePartitioning, dataset___HivePartitioning(schema))
}
-#' Construct a Hive partitioning
+#' Construct Hive partitioning
#'
#' Hive partitioning embeds field names and values in path segments, such as
-#' "/year=2019/month=2/data.parquet". A [HivePartitioning][Partitioning]
-#' is used to parse that in Dataset creation.
+#' "/year=2019/month=2/data.parquet".
#'
#' Because fields are named in the path segments, order of fields passed to
#' `hive_partition()` does not matter.
#' @param ... named list of [data types][data-type], passed to [schema()]
-#' @return A `HivePartitioning`, or a `HivePartitioningFactory` if
+#' @return A [HivePartitioning][Partitioning], or a `HivePartitioningFactory`
if
#' calling `hive_partition()` with no arguments.
#' @examples
#' \donttest{
diff --git a/r/R/dplyr.R b/r/R/dplyr.R
index 0415a95..593c59f 100644
--- a/r/R/dplyr.R
+++ b/r/R/dplyr.R
@@ -47,6 +47,26 @@ arrow_dplyr_query <- function(.data) {
)
}
+#' @export
+print.arrow_dplyr_query <- function(x, ...) {
+ schm <- x$.data$schema
+ cols <- x$selected_columns
+ fields <- map_chr(cols, ~schm$GetFieldByName(.)$ToString())
+ # Strip off the field names as they are in the dataset and add the renamed
ones
+ fields <- paste(names(cols), sub("^.*?: ", "", fields), sep = ": ", collapse
= "\n")
+ cat(class(x$.data)[1], " (query)\n", sep = "")
+ cat(fields, "\n", sep = "")
+ cat("\n")
+ if (!isTRUE(x$filtered_rows)) {
+ cat("* Filter: ", x$filtered_rows$ToString(), "\n", sep = "")
+ }
+ if (length(x$group_by_vars)) {
+ cat("* Grouped by ", paste(x$group_by_vars, collapse = ", "), "\n", sep =
"")
+ }
+ cat("See $.data for the source Arrow object\n")
+ invisible(x)
+}
+
# These are the names reflecting all select/rename, not what is in Arrow
names.arrow_dplyr_query <- function(x) names(x$selected_columns)
diff --git a/r/R/schema.R b/r/R/schema.R
index b492bbc..3a274c8 100644
--- a/r/R/schema.R
+++ b/r/R/schema.R
@@ -33,14 +33,14 @@
#' s <- schema(...)
#'
#' s$ToString()
-#' s$num_fields()
+#' s$num_fields
#' s$field(i)
#' ```
#'
#' @section Methods:
#'
#' - `$ToString()`: convert to a string
-#' - `$num_fields()`: returns the number of fields
+#' - `$num_fields`: returns the number of fields
#' - `$field(i)`: returns the field at index `i` (0-based)
#'
#' @rdname Schema
@@ -49,60 +49,60 @@
Schema <- R6Class("Schema",
inherit = Object,
public = list(
- ToString = function() prettier_dictionary_type(Schema__ToString(self)),
- num_fields = function() Schema__num_fields(self),
+ ToString = function() {
+ fields <- print_schema_fields(self)
+ if (self$HasMetadata) {
+ fields <- paste0(fields, "\n\nSee $metadata for additional Schema
metadata")
+ }
+ fields
+ },
field = function(i) shared_ptr(Field, Schema__field(self, i)),
+ GetFieldByName = function(x) shared_ptr(Field,
Schema__GetFieldByName(self, x)),
serialize = function() Schema__serialize(self),
- Equals = function(other, check_metadata = TRUE) Schema__Equals(self,
other, isTRUE(check_metadata))
+ Equals = function(other, check_metadata = TRUE) {
+ Schema__Equals(self, other, isTRUE(check_metadata))
+ }
),
active = list(
- names = function() Schema__names(self)
+ names = function() Schema__field_names(self),
+ num_fields = function() Schema__num_fields(self),
+ fields = function() map(Schema__fields(self), shared_ptr, class = Field),
+ metadata = function() Schema__metadata(self),
+ HasMetadata = function() Schema__HasMetadata(self)
)
)
Schema$create <- function(...) shared_ptr(Schema, schema_(.fields(list2(...))))
+print_schema_fields <- function(s) {
+ # Alternative to Schema__ToString that doesn't print metadata
+ paste(map_chr(s$fields, ~.$ToString()), collapse = "\n")
+}
+
#' @param ... named list of [data types][data-type]
#' @export
#' @rdname Schema
-# TODO (npr): add examples once ARROW-5505 merges
schema <- Schema$create
#' @export
names.Schema <- function(x) x$names
#' @export
-length.Schema <- function(x) x$num_fields()
+length.Schema <- function(x) x$num_fields
#' read a Schema from a stream
#'
-#' @param stream a stream
+#' @param stream a `Message`, `InputStream`, or `Buffer`
#' @param ... currently ignored
-#'
+#' @return A [Schema]
#' @export
read_schema <- function(stream, ...) {
- UseMethod("read_schema")
-}
-
-#' @export
-read_schema.InputStream <- function(stream, ...) {
- shared_ptr(Schema, ipc___ReadSchema_InputStream(stream))
-}
-
-#' @export
-read_schema.Buffer <- function(stream, ...) {
- stream <- BufferReader$create(stream)
- on.exit(stream$close())
- shared_ptr(Schema, ipc___ReadSchema_InputStream(stream))
-}
-
-#' @export
-read_schema.raw <- function(stream, ...) {
- stream <- BufferReader$create(stream)
- on.exit(stream$close())
- shared_ptr(Schema, ipc___ReadSchema_InputStream(stream))
-}
-
-#' @export
-read_schema.Message <- function(stream, ...) {
- shared_ptr(Schema, ipc___ReadSchema_Message(stream))
+ if (inherits(stream, "Message")) {
+ return(shared_ptr(Schema, ipc___ReadSchema_Message(stream)))
+ } else {
+ if (!inherits(stream, "InputStream")) {
+ stream <- BufferReader$create(stream)
+ on.exit(stream$close())
+ }
+ return(shared_ptr(Schema, ipc___ReadSchema_InputStream(stream)))
+ }
}
diff --git a/r/README.md b/r/README.md
index 5c23fad..1a0da43 100644
--- a/r/README.md
+++ b/r/README.md
@@ -1,16 +1,30 @@
<!-- README.md is generated from README.Rmd. Please edit that file -->
-arrow
-=====
-[](https://cran.r-project.org/package=arrow)
[](https://anaconda.org/conda-forge/r-arrow)
[](https://travis-ci.org/ursa-labs/arrow-r-nightly)
[](
[...]
+# arrow
-[Apache Arrow](https://arrow.apache.org/) is a cross-language development
platform for in-memory data. It specifies a standardized language-independent
columnar memory format for flat and hierarchical data, organized for efficient
analytic operations on modern hardware. It also provides computational
libraries and zero-copy streaming messaging and interprocess communication.
+[](https://cran.r-project.org/package=arrow)
+[](https://anaconda.org/conda-forge/r-arrow)
+[](https://travis-ci.org/ursa-labs/arrow-r-nightly)
+[](https://ci.appveyor.com/project/nealrichardson/arrow-r-nightly-yxl55/branch/master)
+[](https://codecov.io/gh/ursa-labs/arrow-r-nightly)
-The `arrow` package exposes an interface to the Arrow C++ library to access
many of its features in R. This includes support for working with Parquet
(`read_parquet()`, `write_parquet()`) and Feather (`read_feather()`,
`write_feather()`) files, as well as lower-level access to Arrow memory and
messages.
+[Apache Arrow](https://arrow.apache.org/) is a cross-language
+development platform for in-memory data. It specifies a standardized
+language-independent columnar memory format for flat and hierarchical
+data, organized for efficient analytic operations on modern hardware. It
+also provides computational libraries and zero-copy streaming messaging
+and interprocess communication.
-Installation
-------------
+The `arrow` package exposes an interface to the Arrow C++ library to
+access many of its features in R. This includes support for working with
+Parquet (`read_parquet()`, `write_parquet()`) and Feather
+(`read_feather()`, `write_feather()`) files, as well as lower-level
+access to Arrow memory and messages.
+
+## Installation
Install the latest release of `arrow` from CRAN with
@@ -22,20 +36,30 @@ Conda users on Linux and macOS can install `arrow` from
conda-forge with
conda install -c conda-forge r-arrow
-On macOS and Windows, installing a binary package from CRAN will handle
Arrow's C++ dependencies for you. On Linux, unless you use `conda`, the R
package will have to compile its bindings from source and it will need to find
or download the C++ dependencies. As of the 0.16.0 release, this dependency
resolution is automatic on most common Linux distributions. See
`vignette("install", package = "arrow")` for details.
+On macOS and Windows, installing a binary package from CRAN will handle
+Arrow’s C++ dependencies for you. On Linux, unless you use `conda`, the
+R package will have to compile its bindings from source and it will need
+to find or download the C++ dependencies. As of the 0.16.0 release, this
+dependency resolution is automatic on most common Linux distributions.
+See `vignette("install", package = "arrow")` for details.
-If you install the `arrow` package from source and the C++ library is not
found, the R package functions will notify you that Arrow is not available. Call
+If you install the `arrow` package from source and the C++ library is
+not found, the R package functions will notify you that Arrow is not
+available. Call
``` r
arrow::install_arrow()
```
-for version- and platform-specific guidance on installing the Arrow C++
library.
+for version- and platform-specific guidance on installing the Arrow C++
+library.
-When installing from source, if the R and C++ library versions do not match,
installation may fail. If you've previously installed the libraries and want to
upgrade the R package, you'll need to update the Arrow C++ library first.
+When installing from source, if the R and C++ library versions do not
+match, installation may fail. If you’ve previously installed the
+libraries and want to upgrade the R package, you’ll need to update the
+Arrow C++ library first.
-Example
--------
+## Example
``` r
library(arrow)
@@ -88,21 +112,25 @@ as.data.frame(tab)
#> 10 10 0.00231 c
```
-Installing a development version
---------------------------------
+## Installing a development version
-Binary R packages for macOS and Windows are built daily and hosted at
<https://dl.bintray.com/ursalabs/arrow-r/>. To install from there:
+Binary R packages for macOS and Windows are built daily and hosted at
+<https://dl.bintray.com/ursalabs/arrow-r/>. To install from there:
``` r
install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r";)
```
-These daily package builds are not official Apache releases and are not
recommended for production use. They may be useful for testing bug fixes and
new features under active development.
+These daily package builds are not official Apache releases and are not
+recommended for production use. They may be useful for testing bug fixes
+and new features under active development.
-Developing
-----------
+## Developing
-Windows and macOS users who wish to contribute to the R package and don't need
to alter the Arrow C++ library may be able to obtain a recent version of the
library without building from source. On macOS, you may install the C++ library
using [Homebrew](https://brew.sh/):
+Windows and macOS users who wish to contribute to the R package and
+don’t need to alter the Arrow C++ library may be able to obtain a
+recent version of the library without building from source. On macOS,
+you may install the C++ library using [Homebrew](https://brew.sh/):
``` shell
# For the released version:
@@ -111,15 +139,34 @@ brew install apache-arrow
brew install apache-arrow --HEAD
```
-On Windows, you can download a .zip file with the arrow dependencies from the
[rwinlib](https://github.com/rwinlib/arrow/releases) project, and then set the
`RWINLIB_LOCAL` environment variable to point to that zip file before
installing the `arrow` R package. That project contains released versions of
the C++ library; for a development version, Windows users may be able to find a
binary by going to the [Apache Arrow project's
Appveyor](https://ci.appveyor.com/project/ApacheSoftwareFound [...]
-
-If you need to alter both the Arrow C++ library and the R package code, or if
you can't get a binary version of the latest C++ library elsewhere, you'll need
to build it from source too.
-
-First, install the C++ library. See the [C++ developer
guide](https://arrow.apache.org/docs/developers/cpp.html) for details.
-
-Note that after any change to the C++ library, you must reinstall it and run
`make clean` or `git clean -fdx .` to remove any cached object code in the
`r/src/` directory before reinstalling the R package. This is only necessary if
you make changes to the C++ library source; you do not need to manually purge
object files if you are only editing R or Rcpp code inside `r/`.
-
-Once you've built the C++ library, you can install the R package and its
dependencies, along with additional dev dependencies, from the git checkout:
+On Windows, you can download a .zip file with the arrow dependencies
+from the [rwinlib](https://github.com/rwinlib/arrow/releases) project,
+and then set the `RWINLIB_LOCAL` environment variable to point to that
+zip file before installing the `arrow` R package. That project contains
+released versions of the C++ library; for a development version, Windows
+users may be able to find a binary by going to the [Apache Arrow
+project’s
+Appveyor](https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow),
+selecting an R job from a recent build, and downloading the
+`build\arrow-*.zip` file from the “Artifacts” tab.
+
+If you need to alter both the Arrow C++ library and the R package code,
+or if you can’t get a binary version of the latest C++ library
+elsewhere, you’ll need to build it from source too.
+
+First, install the C++ library. See the [C++ developer
+guide](https://arrow.apache.org/docs/developers/cpp.html) for details.
+
+Note that after any change to the C++ library, you must reinstall it and
+run `make clean` or `git clean -fdx .` to remove any cached object code
+in the `r/src/` directory before reinstalling the R package. This is
+only necessary if you make changes to the C++ library source; you do not
+need to manually purge object files if you are only editing R or Rcpp
+code inside `r/`.
+
+Once you’ve built the C++ library, you can install the R package and its
+dependencies, along with additional dev dependencies, from the git
+checkout:
``` shell
cd ../../r
@@ -127,7 +174,10 @@ R -e 'install.packages(c("devtools", "roxygen2",
"pkgdown", "covr")); devtools::
R CMD INSTALL .
```
-If you need to set any compilation flags while building the Rcpp extensions,
you can use the `ARROW_R_CXXFLAGS` environment variable. For example, if you
are using `perf` to profile the R extensions, you may need to set
+If you need to set any compilation flags while building the Rcpp
+extensions, you can use the `ARROW_R_CXXFLAGS` environment variable. For
+example, if you are using `perf` to profile the R extensions, you may
+need to set
``` shell
export ARROW_R_CXXFLAGS=-fno-omit-frame-pointer
@@ -140,13 +190,22 @@ If the package fails to install/load with an error like
this:
unable to load shared object
'/Users/you/R/00LOCK-r/00new/arrow/libs/arrow.so':
dlopen(/Users/you/R/00LOCK-r/00new/arrow/libs/arrow.so, 6): Library not
loaded: @rpath/libarrow.14.dylib
-try setting the environment variable `R_LD_LIBRARY_PATH` to wherever Arrow C++
was put in `make install`, e.g. `export R_LD_LIBRARY_PATH=/usr/local/lib`, and
retry installing the R package.
+try setting the environment variable `R_LD_LIBRARY_PATH` to wherever
+Arrow C++ was put in `make install`, e.g. `export
+R_LD_LIBRARY_PATH=/usr/local/lib`, and retry installing the R package.
-For any other build/configuration challenges, see the [C++ developer
guide](https://arrow.apache.org/docs/developers/cpp.html#building) and
`vignette("install", package = "arrow")`.
+For any other build/configuration challenges, see the [C++ developer
+guide](https://arrow.apache.org/docs/developers/cpp.html#building) and
+`vignette("install", package = "arrow")`.
### Editing Rcpp code
-The `arrow` package uses some customized tools on top of `Rcpp` to prepare its
C++ code in `src/`. If you change C++ code in the R package, you will need to
set the `ARROW_R_DEV` environment variable to `TRUE` (optionally, add it to
your`~/.Renviron` file to persist across sessions) so that the
`data-raw/codegen.R` file is used for code generation.
+The `arrow` package uses some customized tools on top of `Rcpp` to
+prepare its C++ code in `src/`. If you change C++ code in the R package,
+you will need to set the `ARROW_R_DEV` environment variable to `TRUE`
+(optionally, add it to your`~/.Renviron` file to persist across
+sessions) so that the `data-raw/codegen.R` file is used for code
+generation.
The codegen.R script has these additional dependencies:
@@ -163,7 +222,11 @@ Fix any style issues before committing with
./lint.sh --fix
-The lint script requires Python 3 and `clang-format-7`. If the command isn't
found, you can explicitly provide the path to it like `CLANG_FORMAT=$(which
clang-format-7) ./lint.sh`. On macOS, you can get this by installing LLVM via
Homebrew and running the script as `CLANG_FORMAT=$(brew --prefix
llvm@7)/bin/clang-format ./lint.sh`
+The lint script requires Python 3 and `clang-format-7`. If the command
+isn’t found, you can explicitly provide the path to it like
+`CLANG_FORMAT=$(which clang-format-7) ./lint.sh`. On macOS, you can get
+this by installing LLVM via Homebrew and running the script as
+`CLANG_FORMAT=$(brew --prefix llvm@7)/bin/clang-format ./lint.sh`
### Useful functions
@@ -179,7 +242,9 @@ devtools::check() # All package checks; see also below
covr::package_coverage() # See test coverage statistics
```
-Any of those can be run from the command line by wrapping them in `R -e
'$COMMAND'`. There's also a `Makefile` to help with some common tasks from the
command line (`make test`, `make doc`, `make clean`, etc.)
+Any of those can be run from the command line by wrapping them in `R -e
+'$COMMAND'`. There’s also a `Makefile` to help with some common tasks
+from the command line (`make test`, `make doc`, `make clean`, etc.)
### Full package validation
diff --git a/r/man/Partitioning.Rd b/r/man/Partitioning.Rd
index 9eb7b2a..e2b5821 100644
--- a/r/man/Partitioning.Rd
+++ b/r/man/Partitioning.Rd
@@ -4,18 +4,18 @@
\alias{Partitioning}
\alias{DirectoryPartitioning}
\alias{HivePartitioning}
-\title{Define a partitioning for a Source}
+\title{Define Partitioning for a Source}
\description{
-Pass a \code{Partitioning} to a \link{FileSystemSourceFactory}'s
\verb{$create()}
+Pass a \code{Partitioning} object to a \link{FileSystemSourceFactory}'s
\verb{$create()}
method to indicate how the file's paths should be interpreted to define
partitioning.
-A \code{DirectoryPartitioning} describes how to interpret raw path segments, in
+\code{DirectoryPartitioning} describes how to interpret raw path segments, in
order. For example, \code{schema(year = int16(), month = int8())} would define
partitions for file paths like "2019/01/file.parquet",
"2019/02/file.parquet", etc.
-A \code{HivePartitioning} is for Hive-style partitioning, which embeds field
+\code{HivePartitioning} is for Hive-style partitioning, which embeds field
names and values in path segments, such as
"/year=2019/month=2/data.parquet". Because fields are named in the path
segments, order does not matter.
diff --git a/r/man/Schema.Rd b/r/man/Schema.Rd
index e0a17d9..37b2111 100644
--- a/r/man/Schema.Rd
+++ b/r/man/Schema.Rd
@@ -21,7 +21,7 @@ specific numeric precision.
\preformatted{s <- schema(...)
s$ToString()
-s$num_fields()
+s$num_fields
s$field(i)
}
}
@@ -30,7 +30,7 @@ s$field(i)
\itemize{
\item \verb{$ToString()}: convert to a string
-\item \verb{$num_fields()}: returns the number of fields
+\item \verb{$num_fields}: returns the number of fields
\item \verb{$field(i)}: returns the field at index \code{i} (0-based)
}
}
diff --git a/r/man/hive_partition.Rd b/r/man/hive_partition.Rd
index 36f7636..c5c7f2c 100644
--- a/r/man/hive_partition.Rd
+++ b/r/man/hive_partition.Rd
@@ -2,7 +2,7 @@
% Please edit documentation in R/dataset.R
\name{hive_partition}
\alias{hive_partition}
-\title{Construct a Hive partitioning}
+\title{Construct Hive partitioning}
\usage{
hive_partition(...)
}
@@ -10,13 +10,12 @@ hive_partition(...)
\item{...}{named list of \link[=data-type]{data types}, passed to
\code{\link[=schema]{schema()}}}
}
\value{
-A \code{HivePartitioning}, or a \code{HivePartitioningFactory} if
+A \link[=Partitioning]{HivePartitioning}, or a \code{HivePartitioningFactory}
if
calling \code{hive_partition()} with no arguments.
}
\description{
Hive partitioning embeds field names and values in path segments, such as
-"/year=2019/month=2/data.parquet". A \link[=Partitioning]{HivePartitioning}
-is used to parse that in Dataset creation.
+"/year=2019/month=2/data.parquet".
}
\details{
Because fields are named in the path segments, order of fields passed to
diff --git a/r/man/read_schema.Rd b/r/man/read_schema.Rd
index 1573be2..8738b8a 100644
--- a/r/man/read_schema.Rd
+++ b/r/man/read_schema.Rd
@@ -7,10 +7,13 @@
read_schema(stream, ...)
}
\arguments{
-\item{stream}{a stream}
+\item{stream}{a \code{Message}, \code{InputStream}, or \code{Buffer}}
\item{...}{currently ignored}
}
+\value{
+A \link{Schema}
+}
\description{
read a Schema from a stream
}
diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp
index 5f6c518..7e4433e 100644
--- a/r/src/arrowExports.cpp
+++ b/r/src/arrowExports.cpp
@@ -5254,16 +5254,77 @@ RcppExport SEXP _arrow_Schema__field(SEXP s_sexp, SEXP
i_sexp){
// schema.cpp
#if defined(ARROW_R_WITH_ARROW)
-Rcpp::CharacterVector Schema__names(const std::shared_ptr<arrow::Schema>&
schema);
-RcppExport SEXP _arrow_Schema__names(SEXP schema_sexp){
+std::shared_ptr<arrow::Field> Schema__GetFieldByName(const
std::shared_ptr<arrow::Schema>& s, std::string x);
+RcppExport SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
+BEGIN_RCPP
+ Rcpp::traits::input_parameter<const
std::shared_ptr<arrow::Schema>&>::type s(s_sexp);
+ Rcpp::traits::input_parameter<std::string>::type x(x_sexp);
+ return Rcpp::wrap(Schema__GetFieldByName(s, x));
+END_RCPP
+}
+#else
+RcppExport SEXP _arrow_Schema__GetFieldByName(SEXP s_sexp, SEXP x_sexp){
+ Rf_error("Cannot call Schema__GetFieldByName(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// schema.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::vector<std::shared_ptr<arrow::Field>> Schema__fields(const
std::shared_ptr<arrow::Schema>& schema);
+RcppExport SEXP _arrow_Schema__fields(SEXP schema_sexp){
+BEGIN_RCPP
+ Rcpp::traits::input_parameter<const
std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+ return Rcpp::wrap(Schema__fields(schema));
+END_RCPP
+}
+#else
+RcppExport SEXP _arrow_Schema__fields(SEXP schema_sexp){
+ Rf_error("Cannot call Schema__fields(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// schema.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::vector<std::string> Schema__field_names(const
std::shared_ptr<arrow::Schema>& schema);
+RcppExport SEXP _arrow_Schema__field_names(SEXP schema_sexp){
+BEGIN_RCPP
+ Rcpp::traits::input_parameter<const
std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+ return Rcpp::wrap(Schema__field_names(schema));
+END_RCPP
+}
+#else
+RcppExport SEXP _arrow_Schema__field_names(SEXP schema_sexp){
+ Rf_error("Cannot call Schema__field_names(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// schema.cpp
+#if defined(ARROW_R_WITH_ARROW)
+bool Schema__HasMetadata(const std::shared_ptr<arrow::Schema>& schema);
+RcppExport SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
+BEGIN_RCPP
+ Rcpp::traits::input_parameter<const
std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
+ return Rcpp::wrap(Schema__HasMetadata(schema));
+END_RCPP
+}
+#else
+RcppExport SEXP _arrow_Schema__HasMetadata(SEXP schema_sexp){
+ Rf_error("Cannot call Schema__HasMetadata(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+}
+#endif
+
+// schema.cpp
+#if defined(ARROW_R_WITH_ARROW)
+std::string Schema__metadata(const std::shared_ptr<arrow::Schema>& schema);
+RcppExport SEXP _arrow_Schema__metadata(SEXP schema_sexp){
BEGIN_RCPP
Rcpp::traits::input_parameter<const
std::shared_ptr<arrow::Schema>&>::type schema(schema_sexp);
- return Rcpp::wrap(Schema__names(schema));
+ return Rcpp::wrap(Schema__metadata(schema));
END_RCPP
}
#else
-RcppExport SEXP _arrow_Schema__names(SEXP schema_sexp){
- Rf_error("Cannot call Schema__names(). Please use
arrow::install_arrow() to install required runtime libraries. ");
+RcppExport SEXP _arrow_Schema__metadata(SEXP schema_sexp){
+ Rf_error("Cannot call Schema__metadata(). Please use
arrow::install_arrow() to install required runtime libraries. ");
}
#endif
@@ -5897,7 +5958,11 @@ static const R_CallMethodDef CallEntries[] = {
{ "_arrow_Schema__ToString", (DL_FUNC)
&_arrow_Schema__ToString, 1},
{ "_arrow_Schema__num_fields", (DL_FUNC)
&_arrow_Schema__num_fields, 1},
{ "_arrow_Schema__field", (DL_FUNC) &_arrow_Schema__field, 2},
- { "_arrow_Schema__names", (DL_FUNC) &_arrow_Schema__names, 1},
+ { "_arrow_Schema__GetFieldByName", (DL_FUNC)
&_arrow_Schema__GetFieldByName, 2},
+ { "_arrow_Schema__fields", (DL_FUNC) &_arrow_Schema__fields,
1},
+ { "_arrow_Schema__field_names", (DL_FUNC)
&_arrow_Schema__field_names, 1},
+ { "_arrow_Schema__HasMetadata", (DL_FUNC)
&_arrow_Schema__HasMetadata, 1},
+ { "_arrow_Schema__metadata", (DL_FUNC)
&_arrow_Schema__metadata, 1},
{ "_arrow_Schema__serialize", (DL_FUNC)
&_arrow_Schema__serialize, 1},
{ "_arrow_Schema__Equals", (DL_FUNC) &_arrow_Schema__Equals,
3},
{ "_arrow_Table__from_dataframe", (DL_FUNC)
&_arrow_Table__from_dataframe, 1},
diff --git a/r/src/schema.cpp b/r/src/schema.cpp
index 7870043..1699416 100644
--- a/r/src/schema.cpp
+++ b/r/src/schema.cpp
@@ -45,11 +45,32 @@ std::shared_ptr<arrow::Field> Schema__field(const
std::shared_ptr<arrow::Schema>
}
// [[arrow::export]]
-Rcpp::CharacterVector Schema__names(const std::shared_ptr<arrow::Schema>&
schema) {
- auto fields = schema->fields();
- return Rcpp::CharacterVector(
- fields.begin(), fields.end(),
- [](const std::shared_ptr<arrow::Field>& field) { return field->name();
});
+std::shared_ptr<arrow::Field> Schema__GetFieldByName(
+ const std::shared_ptr<arrow::Schema>& s, std::string x) {
+ return s->GetFieldByName(x);
+}
+
+// [[arrow::export]]
+std::vector<std::shared_ptr<arrow::Field>> Schema__fields(
+ const std::shared_ptr<arrow::Schema>& schema) {
+ return schema->fields();
+}
+
+// [[arrow::export]]
+std::vector<std::string> Schema__field_names(
+ const std::shared_ptr<arrow::Schema>& schema) {
+ return schema->field_names();
+}
+
+// [[arrow::export]]
+bool Schema__HasMetadata(const std::shared_ptr<arrow::Schema>& schema) {
+ return schema->HasMetadata();
+}
+
+// [[arrow::export]]
+std::string Schema__metadata(const std::shared_ptr<arrow::Schema>& schema) {
+ // TODO: return a useful object, not just ToString?
+ return schema->metadata()->ToString();
}
// [[arrow::export]]
diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R
index 7e6fe09..adfcd41 100644
--- a/r/tests/testthat/test-dataset.R
+++ b/r/tests/testthat/test-dataset.R
@@ -252,6 +252,58 @@ test_that("dplyr method not implemented messages", {
expect_not_implemented(ds %>% filter(int == 1) %>% summarize(n()))
})
+test_that("Dataset and query print methods", {
+ ds <- open_dataset(hive_dir)
+ expect_output(
+ print(ds),
+ paste(
+ "Dataset",
+ "int: int32",
+ "dbl: double",
+ "lgl: bool",
+ "chr: string",
+ "fct: dictionary<values=string, indices=int32>",
+ "ts: timestamp[us, tz=GMT]",
+ "group: int32",
+ "other: string",
+ "",
+ "See $metadata for additional Schema metadata",
+ sep = "\n"
+ ),
+ fixed = TRUE
+ )
+ expect_is(ds$metadata, "character")
+ q <- select(ds, string = chr, lgl, integer = int)
+ expect_output(
+ print(q),
+ paste(
+ "Dataset (query)",
+ "string: string",
+ "lgl: bool",
+ "integer: int32",
+ "",
+ "See $.data for the source Arrow object",
+ sep = "\n"
+ ),
+ fixed = TRUE
+ )
+ expect_output(
+ print(q %>% filter(integer == 6) %>% group_by(lgl)),
+ paste(
+ "Dataset (query)",
+ "string: string",
+ "lgl: bool",
+ "integer: int32",
+ "",
+ "* Filter: (int == 6:double)",
+ "* Grouped by lgl",
+ "See $.data for the source Arrow object",
+ sep = "\n"
+ ),
+ fixed = TRUE
+ )
+})
+
test_that("Assembling a Dataset manually and getting a Table", {
fs <- LocalFileSystem$create()
selector <- FileSelector$create(dataset_dir, recursive = TRUE)
diff --git a/r/tests/testthat/test-schema.R b/r/tests/testthat/test-schema.R
index 9c9bd7a..8e07cd8 100644
--- a/r/tests/testthat/test-schema.R
+++ b/r/tests/testthat/test-schema.R
@@ -25,6 +25,20 @@ test_that("Alternate type names are supported", {
expect_equal(names(schema(b = double(), c = bool(), d = string())), c("b",
"c", "d"))
})
+test_that("Schema print method", {
+ expect_output(
+ print(schema(b = double(), c = bool(), d = string())),
+ paste(
+ "Schema",
+ "b: double",
+ "c: bool",
+ "d: string",
+ sep = "\n"
+ ),
+ fixed = TRUE
+ )
+})
+
test_that("reading schema from Buffer", {
# TODO: this uses the streaming format, i.e. from RecordBatchStreamWriter
# maybe there is an easier way to serialize a schema
diff --git a/r/vignettes/dataset.Rmd b/r/vignettes/dataset.Rmd
new file mode 100644
index 0000000..4b9def6
--- /dev/null
+++ b/r/vignettes/dataset.Rmd
@@ -0,0 +1,260 @@
+---
+title: "Working with Arrow Datasets and dplyr"
+description: ""
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{Working with Arrow Datasets and dplyr}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+Apache Arrow lets you work efficiently with large, multi-file datasets.
+The `arrow` R package provides a `dplyr` interface to Arrow Datasets,
+as well as other tools for interactive exploration of Arrow data.
+
+This vignette introduces Datasets and shows how to use `dplyr` to analyze them.
+It describes both what is possible to do with Arrow now
+and what is on the immediate development roadmap.
+
+## Example: NYC taxi data
+
+The [New York City taxi trip record
data](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page)
+is widely used in big data exercises and competitions.
+For demonstration purposes, we have hosted a Parquet-formatted version
+of about 10 years of the trip data in a public S3 bucket.
+
+In a future release, you'll be able to point your R session at S3 and query
+the dataset from there. For now, datasets need to be on your local file system.
+To download the files,
+
+```r
+bucket <- "https://ursa-labs-taxi-data.s3.us-east-2.amazonaws.com";
+dir.create("nyc-taxi")
+for (year in 2009:2019) {
+ for (month in 1:12) {
+ if (month < 10) {
+ month <- paste0("0", month)
+ }
+ try(download.file(
+ paste(bucket, year, month, "data.parquet", sep = "/"),
+ file.path("nyc-taxi", year, month, "data.parquet")
+ ))
+ }
+}
+```
+
+It is expected that some files will not download because they do not
exist--December 2019,
+for example--hence the `try()`.
+The total file size is around 37 gigabytes, even in the efficient Parquet file
format.
+That's bigger than memory on most people's computers,
+so we can't just read it all in and stack it into a single data frame.
+
+Given the size, if you're running this locally and don't have a fast
connection,
+feel free to grab only a year or two of data.
+
+## Getting started
+
+Because `dplyr` is not necessary for many Arrow workflows,
+it is an optional (`Suggests`) dependency. So, to work with Datasets,
+we need to load both `arrow` and `dplyr`.
+
+```r
+library(arrow)
+library(dplyr)
+```
+
+The first step is to create our Dataset object, pointing at the directory of
data.
+
+```r
+ds <- open_dataset("nyc-taxi", partitioning = c("year", "month"))
+```
+
+The default file format for `open_dataset()` is Parquet; if we had a directory
+of Arrow format files, we could include `format = "arrow"` in the call.
+Future versions will support more file formats, including CSV/delimited text
data
+and JSON.
+
+The `partitioning` argument lets us specify how the file paths provide
information
+about how the dataset is chunked into different files. Our files in this
example
+have file paths like
+
+```
+2009/01/data.parquet
+2009/02/data.parquet
+...
+```
+
+By providing a character vector to `partitioning`, we're saying that the first
+path segment gives the value for "year" and the second segment is "month".
+Every row in `2009/01/data.parquet` has a value of 2009 for "year"
+and 1 for "month", even though those columns may not actually be present in
the file.
+
+Indeed, when we look at the dataset, we see that in addition to the columns
present
+in every file, there are also columns "year" and "month".
+
+```
+ds
+
+## Dataset
+## vendor_id: string
+## pickup_at: timestamp[us]
+## dropoff_at: timestamp[us]
+## passenger_count: int8
+## trip_distance: float
+## pickup_longitude: float
+## pickup_latitude: float
+## rate_code_id: string
+## store_and_fwd_flag: string
+## dropoff_longitude: float
+## dropoff_latitude: float
+## payment_type: string
+## fare_amount: float
+## extra: float
+## mta_tax: float
+## tip_amount: float
+## tolls_amount: float
+## total_amount: float
+## improvement_surcharge: float
+## pickup_location_id: int32
+## dropoff_location_id: int32
+## congestion_surcharge: float
+## year: int32
+## month: int32
+
+See $metadata for additional Schema metadata
+```
+
+The other form of partitioning currently supported is
[Hive](https://hive.apache.org/)-style,
+in which the partition variable names are included in the path segments.
+If we had saved our files in paths like
+
+```
+year=2009/month=01/data.parquet
+year=2009/month=02/data.parquet
+...
+```
+
+we would not have had to provide the names in `partitioning`:
+we could have just called `ds <- open_dataset("nyc-taxi")` and the partitions
+would have been detected automatically.
+
+## Querying the dataset
+
+Up to this point, we haven't loaded any data: we have walked directories to
find
+files, we've parsed file paths to identify partitions, and we've read the
+headers of the Parquet files to inspect their schemas so that we can make sure
+they all line up.
+
+In the current release, `arrow` supports methods for selecting a window of
data:
+`select()`, `rename()`, and `filter()`. Aggregation is not yet supported,
+nor is deriving or projecting new columns, so before you call `summarize()` or
+`mutate()`, you'll need to `collect()` the data first,
+which pulls your selected window of data into an in-memory R data frame.
+While we could have made those methods `collect()` the data they needed
+automatically and invisibly to the end user,
+we thought it best to make it explicit when you're pulling data into memory
+so that you can construct your queries most efficiently
+and not be surprised when some query consumes way more resources than expected.
+
+Here's an example. Suppose I was curious about tipping behavior among the
+longest taxi rides. Let's find the median tip percentage for rides with
+fares greater than $100 in 2015, broken down by the number of passengers:
+
+```r
+system.time(ds %>%
+ filter(total_amount > 100, year == 2015) %>%
+ select(tip_amount, total_amount, passenger_count) %>%
+ group_by(passenger_count) %>%
+ collect() %>%
+ summarize(
+ tip_pct = median(100 * tip_amount / total_amount),
+ n = n()
+ ) %>%
+ print())
+```
+
+```
+## # A tibble: 10 x 3
+## passenger_count tip_pct n
+## <int> <dbl> <int>
+## 1 0 9.84 380
+## 2 1 16.7 143087
+## 3 2 16.6 34418
+## 4 3 14.4 8922
+## 5 4 11.4 4771
+## 6 5 16.7 5806
+## 7 6 16.7 3338
+## 8 7 16.7 11
+## 9 8 16.7 32
+## 10 9 16.7 42
+##
+## user system elapsed
+## 25.227 1.162 3.767
+```
+
+We just selected a window out of a dataset with around 2 billion rows
+and aggregated on it in under 4 seconds on my laptop. How does this work?
+
+First, `select()`/`rename()`, `filter()`, and `group_by()`
+record their actions but don't evaluate on the data until you run `collect()`.
+
+```r
+ds %>%
+ filter(total_amount > 100, year == 2015) %>%
+ select(tip_amount, total_amount, passenger_count) %>%
+ group_by(passenger_count)
+```
+
+```
+## Dataset (query)
+## tip_amount: float
+## total_amount: float
+## passenger_count: int8
+##
+## * Filter: ((total_amount > 100:double) and (year == 2015:double))
+## * Grouped by passenger_count
+## See $.data for the source Arrow object
+```
+
+This returns instantly and shows the window selection you've made, without
+loading data from the files. Because the evaluation of these queries is
deferred,
+you can build up a query that selects down to a small window without generating
+intermediate datasets that would potentially be large.
+
+Second, all work is pushed down to the individual data files,
+and depending on the file format, chunks of data within the files. As a result,
+we can select a window of data from a much larger dataset by collecting the
+smaller slices from each file--we don't have to load the whole dataset in
memory
+in order to slice from it.
+
+Third, because of partitioning, we can ignore some files entirely.
+In this example, by filtering `year == 2015`, all files corresponding to other
years
+are immediately excluded: we don't have to load them in order to find that no
+rows match the filter. Relatedly, since Parquet files contain row groups with
+statistics on the data within, there may be entire chunks of data we can
+avoid scanning because they have no rows where `total_amount > 100`.
+
+## Going farther
+
+There are a few ways you can control the Dataset creation to adapt to special
use cases.
+For one, you can specify a `schema` argument to declare the columns and their
data types.
+This is useful if you have data files that have different storage schema
+(for example, a column could be `int32` in one and `int8` in another)
+and you want to ensure that the resulting Dataset has a specific type.
+To be clear, it's not necessary to specify a schema, even in this example of
+mixed integer types, because the Dataset constructor will reconcile
differences like these.
+The schema specification just lets you declare what you want the result to be.
+
+Similarly, you can provide a Schema in the `partitioning` argument of
`open_dataset()`
+in order to declare the types of the virtual columns that define the
partitions.
+This would be useful, in our taxi dataset example, if you wanted to keep
+"month" as a string instead of an integer for some reason.
+
+Another feature of Datasets is that they can be composed of multiple data
sources.
+That is, you may have a directory of partitioned Parquet files in one location,
+and in another directory, files that haven't been partitioned.
+In the future, when there is support for cloud storage and other file formats,
+this would mean you could point to an S3 bucked of Parquet data and a directory
+of CSVs on the local file system and query them together as a single dataset.
+To create a multi-source dataset, provide a list of sources to `open_dataset()`
+instead of a file path. See `?open_source` for creating data sources.
