pitrou commented on a change in pull request #12590:
URL: https://github.com/apache/arrow/pull/12590#discussion_r837484456
##########
File path: cpp/examples/arrow/CMakeLists.txt
##########
@@ -133,4 +133,11 @@ if(ARROW_PARQUET AND ARROW_DATASET)
add_arrow_example(join_example EXTRA_LINK_LIBS ${DATASET_EXAMPLES_LINK_LIBS})
add_dependencies(join-example parquet)
+
+ add_arrow_example(udf_example EXTRA_LINK_LIBS ${DATASET_EXAMPLES_LINK_LIBS})
+ add_dependencies(udf-example parquet)
+
+ add_arrow_example(aggregate_example EXTRA_LINK_LIBS
${DATASET_EXAMPLES_LINK_LIBS})
+ add_dependencies(aggregate-example parquet)
Review comment:
Why the Parquet dependencies?
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
Review comment:
Can we reserve all-caps for macros?
```suggestion
using ArrowArrayType = typename arrow::TypeTraits<TYPE>::ArrayType;
using ArrowBuilderType = typename arrow::TypeTraits<TYPE>::BuilderType;
```
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
+}
+
+class UDFOptionsType : public cp::FunctionOptionsType {
Review comment:
Can we give this a meaningful name? For example "TernarySumOptionsType"?
##########
File path: cpp/src/arrow/python/udf.h
##########
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "arrow/python/platform.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/compute/api_scalar.h"
+#include "arrow/compute/cast.h"
+#include "arrow/compute/exec.h"
+#include "arrow/compute/function.h"
+#include "arrow/compute/registry.h"
+#include "arrow/datum.h"
+#include "arrow/util/cpu_info.h"
+#include "arrow/util/logging.h"
+
+#include "arrow/python/common.h"
+#include "arrow/python/pyarrow.h"
+#include "arrow/python/visibility.h"
+
+namespace cp = arrow::compute;
Review comment:
It's not recommended to declare namespace aliases in `.h` files, they
might crash with third-party code.
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
Review comment:
Nit: we know it's a lambda, no need to suffix it with "lambda"...
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
+ if (PyCallable_Check(py_function)) {
Review comment:
Same here: check this on construction.
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
Review comment:
```suggestion
return arrow::Table::Make(schema, data_vector, /*num_rows=*/6);
```
##########
File path: python/pyarrow/_compute.pyx
##########
@@ -2182,3 +2322,257 @@ cdef CExpression _bind(Expression filter, Schema
schema) except *:
return GetResultValue(filter.unwrap().Bind(
deref(pyarrow_unwrap_schema(schema).get())))
+
+
+cdef CFunctionDoc _make_function_doc(func_doc):
+ """
+ Helper function to generate the FunctionDoc
+ """
+ cdef:
+ CFunctionDoc f_doc
+ vector[c_string] c_arg_names
+ c_bool c_options_required
+ if func_doc and isinstance(func_doc, dict):
+ if func_doc["summary"] and isinstance(func_doc["summary"], str):
+ f_doc.summary = func_doc["summary"].encode()
+ else:
+ raise ValueError("key `summary` cannot be None")
+
+ if func_doc["description"] and isinstance(func_doc["description"],
str):
+ f_doc.description = func_doc["description"].encode()
+ else:
+ raise ValueError("key `description` cannot be None")
+
+ if func_doc["arg_names"] and isinstance(func_doc["arg_names"], list):
+ for arg_name in func_doc["arg_names"]:
+ if isinstance(arg_name, str):
+ c_arg_names.push_back(arg_name.encode())
+ else:
+ raise ValueError(
+ "key `arg_names` must be a list of strings")
+ f_doc.arg_names = c_arg_names
+ else:
+ raise ValueError("key `arg_names` cannot be None")
+
+ # UDFOptions integration:
+ # TODO: https://issues.apache.org/jira/browse/ARROW-16041
+ f_doc.options_class = tobytes("None")
+
+ c_options_required = False
+ f_doc.options_required = c_options_required
+
+ return f_doc
+ else:
+ raise ValueError(f"func_doc must be a dictionary")
+
+
+cdef class UDFError(Exception):
Review comment:
I don't think it's useful to define custom exceptions here, just use
built-in ones.
##########
File path: python/pyarrow/_compute.pyx
##########
@@ -199,6 +202,135 @@ FunctionDoc = namedtuple(
"options_required"))
+cdef wrap_arity(const CArity c_arity):
+ """
+ Wrap a C++ Arity in an Arity object
+ """
+ cdef Arity arity = Arity.__new__(Arity)
+ arity.init(c_arity)
+ return arity
+
+
+cdef wrap_input_type(const CInputType c_input_type):
+ """
+ Wrap a C++ InputType in an InputType object
+ """
+ cdef InputType input_type = InputType.__new__(InputType)
+ input_type.init(c_input_type)
+ return input_type
+
+
+cdef class InputType(_Weakrefable):
+ """
+ An interface for defining input-types for streaming execution engine
+ applications.
+ """
+
+ def __init__(self):
+ raise TypeError("Cannot use constructor to initialize InputType")
+
+ cdef void init(self, const CInputType &input_type):
+ self.input_type = input_type
+
+ @staticmethod
+ def scalar(data_type):
+ """
+ create a scalar input type of the given data type
+
+ Parameter
+ ---------
+ data_type: DataType
+
+ Examples
+ --------
+
+ >>> import pyarrow as pa
+ >>> from pyarrow.compute import InputType
+ >>> in_type = InputType.scalar(pa.int32())
+ <pyarrow._compute.InputType object at 0x1029fdcb0>
+ """
+ cdef:
+ shared_ptr[CDataType] c_data_type
+ CInputType c_input_type
+ c_data_type = pyarrow_unwrap_data_type(data_type)
+ c_input_type = CInputType.Scalar(c_data_type)
+ return wrap_input_type(c_input_type)
+
+ @staticmethod
+ def array(data_type):
+ """
+ create an array input type of the given data type
+
+ Parameter
+ ---------
+ data_type: DataType
+
+ Examples
+ --------
+
+ >>> import pyarrow as pa
+ >>> from pyarrow.compute import InputType
+ >>> in_type = InputType.array(pa.int32())
+ <pyarrow._compute.InputType object at 0x102ba4850>
+ """
+ cdef:
+ shared_ptr[CDataType] c_data_type
+ CInputType c_input_type
+ c_data_type = pyarrow_unwrap_data_type(data_type)
+ c_input_type = CInputType.Array(c_data_type)
+ return wrap_input_type(c_input_type)
+
+
+cdef class Arity(_Weakrefable):
Review comment:
This doesn't seem resolved to me. I agree this custom class is
completely overkill. Also, it doesn't match how this information is currently
exposed:
```python
>>> import pyarrow as pa, pyarrow.compute as pc
>>> pc.get_function("random")
arrow.compute.Function<name=random, kind=scalar, arity=0, num_kernels=1>
>>> pc.get_function("sort_indices")
arrow.compute.Function<name=sort_indices, kind=meta, arity=1, num_kernels=0>
>>> pc.get_function("add")
arrow.compute.Function<name=add, kind=scalar, arity=2, num_kernels=33>
>>> pc.get_function("if_else")
arrow.compute.Function<name=if_else, kind=scalar, arity=3, num_kernels=39>
>>> pc.get_function("choose")
arrow.compute.Function<name=choose, kind=scalar, arity=Ellipsis,
num_kernels=32>
```
(note the varargs convention of returning Ellipsis is perhaps not enough if
we want to faithfully mirror the C++ function metadata, but that can be a
followup PR)
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
+}
+
+class UDFOptionsType : public cp::FunctionOptionsType {
+ const char* type_name() const override { return "UDFOptionsType"; }
+ std::string Stringify(const cp::FunctionOptions&) const override {
+ return "UDFOptionsType";
+ }
+ bool Compare(const cp::FunctionOptions& options,
+ const cp::FunctionOptions& other) const override {
+ return true;
+ }
+ std::unique_ptr<cp::FunctionOptions> Copy(
+ const cp::FunctionOptions& options) const override;
+};
+
+cp::FunctionOptionsType* GetUDFOptionsType() {
+ static UDFOptionsType options_type;
+ return &options_type;
+}
+
+class UDFOptions : public cp::FunctionOptions {
Review comment:
It's not very useful that the custom options class doesn't carry any
actual option. Is it useful? Do we want to add something there to spice things
up?
##########
File path: python/pyarrow/public-api.pxi
##########
@@ -38,7 +37,6 @@ cdef api shared_ptr[CBuffer] pyarrow_unwrap_buffer(object
buffer):
return shared_ptr[CBuffer]()
-
Review comment:
Why these changes?
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
Review comment:
Can instead write:
```suggestion
ARROW_RETURN_NOT_OK(builder.AppendValues(values));
return builder.Finish();
```
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
+}
+
+class UDFOptionsType : public cp::FunctionOptionsType {
+ const char* type_name() const override { return "UDFOptionsType"; }
+ std::string Stringify(const cp::FunctionOptions&) const override {
+ return "UDFOptionsType";
+ }
+ bool Compare(const cp::FunctionOptions& options,
+ const cp::FunctionOptions& other) const override {
+ return true;
+ }
+ std::unique_ptr<cp::FunctionOptions> Copy(
+ const cp::FunctionOptions& options) const override;
+};
+
+cp::FunctionOptionsType* GetUDFOptionsType() {
+ static UDFOptionsType options_type;
+ return &options_type;
+}
+
+class UDFOptions : public cp::FunctionOptions {
+ public:
+ UDFOptions() : cp::FunctionOptions(GetUDFOptionsType()) {}
+};
+
+std::unique_ptr<cp::FunctionOptions> UDFOptionsType::Copy(
+ const cp::FunctionOptions&) const {
+ return std::unique_ptr<cp::FunctionOptions>(new UDFOptions());
+}
+
+class ExampleNodeOptions : public cp::ExecNodeOptions {};
+
+// a basic ExecNode which ignores all input batches
+class ExampleNode : public cp::ExecNode {
+ public:
+ ExampleNode(ExecNode* input, const ExampleNodeOptions&)
+ : ExecNode(/*plan=*/input->plan(), /*inputs=*/{input},
+ /*input_labels=*/{"ignored"},
+ /*output_schema=*/input->output_schema(), /*num_outputs=*/1)
{}
+
+ const char* kind_name() const override { return "ExampleNode"; }
+
+ arrow::Status StartProducing() override {
+ outputs_[0]->InputFinished(this, 0);
+ return arrow::Status::OK();
+ }
+
+ void ResumeProducing(ExecNode* output) override {}
+ void PauseProducing(ExecNode* output) override {}
+
+ void StopProducing(ExecNode* output) override {
inputs_[0]->StopProducing(this); }
+ void StopProducing() override { inputs_[0]->StopProducing(); }
+
+ void InputReceived(ExecNode* input, cp::ExecBatch batch) override {}
+ void ErrorReceived(ExecNode* input, arrow::Status error) override {}
+ void InputFinished(ExecNode* input, int total_batches) override {}
+
+ arrow::Future<> finished() override { return inputs_[0]->finished(); }
+};
+
+arrow::Result<cp::ExecNode*> ExampleExecNodeFactory(cp::ExecPlan* plan,
+ std::vector<cp::ExecNode*>
inputs,
+ const cp::ExecNodeOptions&
options) {
+ const auto& example_options =
+ arrow::internal::checked_cast<const ExampleNodeOptions&>(options);
+
+ return plan->EmplaceNode<ExampleNode>(inputs[0], example_options);
+}
+
+const cp::FunctionDoc func_doc{
+ "User-defined-function usage to demonstrate registering an out-of-tree
function",
+ "returns x + y + z",
+ {"x", "y", "z"},
+ "UDFOptions"};
+
+arrow::Status SampleFunction(cp::KernelContext* ctx, const cp::ExecBatch&
batch,
+ arrow::Datum* out) {
+ auto in_res = cp::CallFunction("add", {batch[0].array(), batch[1].array()});
+ auto in_arr = in_res.ValueOrDie().make_array();
+ auto final_res = cp::CallFunction("add", {in_arr, batch[2].array()});
+ auto final_arr = final_res.ValueOrDie().array();
+ auto datum = new arrow::Datum(final_arr);
+ *out = *datum;
+ return arrow::Status::OK();
+}
+
+arrow::Status Execute() {
+ const std::string name = "add_three";
+ auto func = std::make_shared<cp::ScalarFunction>(name, cp::Arity::Ternary(),
&func_doc);
+
+ auto options = std::make_shared<UDFOptions>();
+ cp::ScalarKernel kernel(
+ {cp::InputType::Array(arrow::int64()),
cp::InputType::Array(arrow::int64()),
+ cp::InputType::Array(arrow::int64())},
+ arrow::int64(), SampleFunction);
+
+ kernel.mem_allocation = cp::MemAllocation::NO_PREALLOCATE;
+ kernel.null_handling = cp::NullHandling::COMPUTED_NO_PREALLOCATE;
+
+ ABORT_ON_FAILURE(func->AddKernel(std::move(kernel)));
Review comment:
This function is returning a `Status`, so you can use
`ARROW_RETURN_NOT_OK` everywhere.
##########
File path: cpp/src/arrow/python/api.h
##########
@@ -28,3 +28,4 @@
#include "arrow/python/numpy_to_arrow.h"
#include "arrow/python/python_to_arrow.h"
#include "arrow/python/serialize.h"
+#include "arrow/python/udf.h"
Review comment:
I don't really think it makes sense to add this. `udf.h` is just an
internal helper to be used from our Cython code, right?
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
+}
+
+class UDFOptionsType : public cp::FunctionOptionsType {
+ const char* type_name() const override { return "UDFOptionsType"; }
+ std::string Stringify(const cp::FunctionOptions&) const override {
+ return "UDFOptionsType";
+ }
+ bool Compare(const cp::FunctionOptions& options,
+ const cp::FunctionOptions& other) const override {
+ return true;
+ }
+ std::unique_ptr<cp::FunctionOptions> Copy(
+ const cp::FunctionOptions& options) const override;
+};
+
+cp::FunctionOptionsType* GetUDFOptionsType() {
+ static UDFOptionsType options_type;
+ return &options_type;
+}
+
+class UDFOptions : public cp::FunctionOptions {
+ public:
+ UDFOptions() : cp::FunctionOptions(GetUDFOptionsType()) {}
+};
+
+std::unique_ptr<cp::FunctionOptions> UDFOptionsType::Copy(
+ const cp::FunctionOptions&) const {
+ return std::unique_ptr<cp::FunctionOptions>(new UDFOptions());
+}
+
+class ExampleNodeOptions : public cp::ExecNodeOptions {};
+
+// a basic ExecNode which ignores all input batches
+class ExampleNode : public cp::ExecNode {
+ public:
+ ExampleNode(ExecNode* input, const ExampleNodeOptions&)
+ : ExecNode(/*plan=*/input->plan(), /*inputs=*/{input},
+ /*input_labels=*/{"ignored"},
+ /*output_schema=*/input->output_schema(), /*num_outputs=*/1)
{}
+
+ const char* kind_name() const override { return "ExampleNode"; }
+
+ arrow::Status StartProducing() override {
+ outputs_[0]->InputFinished(this, 0);
+ return arrow::Status::OK();
+ }
+
+ void ResumeProducing(ExecNode* output) override {}
+ void PauseProducing(ExecNode* output) override {}
+
+ void StopProducing(ExecNode* output) override {
inputs_[0]->StopProducing(this); }
+ void StopProducing() override { inputs_[0]->StopProducing(); }
+
+ void InputReceived(ExecNode* input, cp::ExecBatch batch) override {}
+ void ErrorReceived(ExecNode* input, arrow::Status error) override {}
+ void InputFinished(ExecNode* input, int total_batches) override {}
+
+ arrow::Future<> finished() override { return inputs_[0]->finished(); }
+};
+
+arrow::Result<cp::ExecNode*> ExampleExecNodeFactory(cp::ExecPlan* plan,
+ std::vector<cp::ExecNode*>
inputs,
+ const cp::ExecNodeOptions&
options) {
+ const auto& example_options =
+ arrow::internal::checked_cast<const ExampleNodeOptions&>(options);
+
+ return plan->EmplaceNode<ExampleNode>(inputs[0], example_options);
+}
+
+const cp::FunctionDoc func_doc{
+ "User-defined-function usage to demonstrate registering an out-of-tree
function",
+ "returns x + y + z",
+ {"x", "y", "z"},
+ "UDFOptions"};
+
+arrow::Status SampleFunction(cp::KernelContext* ctx, const cp::ExecBatch&
batch,
+ arrow::Datum* out) {
+ auto in_res = cp::CallFunction("add", {batch[0].array(), batch[1].array()});
+ auto in_arr = in_res.ValueOrDie().make_array();
+ auto final_res = cp::CallFunction("add", {in_arr, batch[2].array()});
+ auto final_arr = final_res.ValueOrDie().array();
+ auto datum = new arrow::Datum(final_arr);
+ *out = *datum;
+ return arrow::Status::OK();
Review comment:
We should promote proper error handling in examples.
Untested suggestion:
```suggestion
// temp = x + y; return temp + z
ARROW_ASSIGN_OR_RAISE(auto temp, cp::CallFunction("add", {batch[0],
batch[1]}));
ARROW_ASSIGN_OR_RAISE(*out, cp::CallFunction("add", {temp, batch[2]}));
return arrow::Status::OK();
```
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
Review comment:
This should be checked above while constructing the scalar function.
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
+ if (PyCallable_Check(py_function)) {
+ RETURN_NOT_OK(VerifyArityAndInput(arity, batch));
+ if (batch[0].is_array()) { // checke 0-th element to select array
callable
Review comment:
I don't understand why you have two different branches here. You could
just wrap the objects underlying the input Datums, regardless of their kind.
##########
File path: cpp/examples/arrow/udf_example.cc
##########
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/compute/api.h>
+#include <arrow/compute/exec/exec_plan.h> // ARROW-15263
+#include <arrow/util/async_generator.h>
+#include <arrow/util/future.h>
+#include <arrow/util/make_unique.h>
+#include <arrow/util/vector.h>
+
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// Demonstrate registering a user-defined Arrow compute function outside of
the Arrow
+// source tree
+
+namespace cp = ::arrow::compute;
+
+#define ABORT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ abort(); \
+ } \
+ } while (0);
+
+template <typename TYPE,
+ typename = typename
std::enable_if<arrow::is_number_type<TYPE>::value |
+
arrow::is_boolean_type<TYPE>::value |
+
arrow::is_temporal_type<TYPE>::value>::type>
+arrow::Result<std::shared_ptr<arrow::Array>> GetArrayDataSample(
+ const std::vector<typename TYPE::c_type>& values) {
+ using ARROW_ARRAY_TYPE = typename arrow::TypeTraits<TYPE>::ArrayType;
+ using ARROW_BUILDER_TYPE = typename arrow::TypeTraits<TYPE>::BuilderType;
+ ARROW_BUILDER_TYPE builder;
+ ARROW_RETURN_NOT_OK(builder.Reserve(values.size()));
+ std::shared_ptr<ARROW_ARRAY_TYPE> array;
+ ARROW_RETURN_NOT_OK(builder.AppendValues(values));
+ ARROW_RETURN_NOT_OK(builder.Finish(&array));
+ return array;
+}
+
+arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
+ std::shared_ptr<arrow::Table> table;
+
+ auto field_vector = {
+ arrow::field("a", arrow::int64()), arrow::field("x", arrow::int64()),
+ arrow::field("y", arrow::int64()), arrow::field("z", arrow::int64()),
+ arrow::field("b", arrow::boolean())};
+
+ ARROW_ASSIGN_OR_RAISE(auto int_array,
+ GetArrayDataSample<arrow::Int64Type>({1, 2, 3, 4, 5,
6}));
+ ARROW_ASSIGN_OR_RAISE(auto x,
+ GetArrayDataSample<arrow::Int64Type>({21, 22, 23, 24,
25, 26}));
+ ARROW_ASSIGN_OR_RAISE(auto y,
+ GetArrayDataSample<arrow::Int64Type>({31, 32, 33, 34,
35, 36}));
+ ARROW_ASSIGN_OR_RAISE(auto z,
+ GetArrayDataSample<arrow::Int64Type>({41, 42, 43, 44,
45, 46}));
+ ARROW_ASSIGN_OR_RAISE(auto bool_array,
GetArrayDataSample<arrow::BooleanType>(
+ {false, true, false, true, true,
false}));
+
+ auto schema = arrow::schema(field_vector);
+ auto data_vector = {int_array, x, y, z, bool_array};
+
+ table = arrow::Table::Make(schema, data_vector, 6);
+
+ return table;
+}
+
+class UDFOptionsType : public cp::FunctionOptionsType {
+ const char* type_name() const override { return "UDFOptionsType"; }
+ std::string Stringify(const cp::FunctionOptions&) const override {
+ return "UDFOptionsType";
+ }
+ bool Compare(const cp::FunctionOptions& options,
+ const cp::FunctionOptions& other) const override {
+ return true;
+ }
+ std::unique_ptr<cp::FunctionOptions> Copy(
+ const cp::FunctionOptions& options) const override;
+};
+
+cp::FunctionOptionsType* GetUDFOptionsType() {
+ static UDFOptionsType options_type;
+ return &options_type;
+}
+
+class UDFOptions : public cp::FunctionOptions {
+ public:
+ UDFOptions() : cp::FunctionOptions(GetUDFOptionsType()) {}
+};
+
+std::unique_ptr<cp::FunctionOptions> UDFOptionsType::Copy(
+ const cp::FunctionOptions&) const {
+ return std::unique_ptr<cp::FunctionOptions>(new UDFOptions());
+}
+
+class ExampleNodeOptions : public cp::ExecNodeOptions {};
+
+// a basic ExecNode which ignores all input batches
+class ExampleNode : public cp::ExecNode {
Review comment:
I'm curious: why does a UDF function example need a custom ExecNode
implementation?
##########
File path: python/pyarrow/_compute.pxd
##########
@@ -21,6 +21,19 @@ from pyarrow.lib cimport *
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
+cdef class Arity(_Weakrefable):
Review comment:
Note that declaring those classes here is only useful if they are being
used from other Cython modules, which doesn't seem to be the case.
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
Review comment:
Our style guide mandates C++-style casts, so `static_cast` here.
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
Review comment:
You need to use `OwnedRefNoGIL` so that the object is actually kept
alive by the lambda function.
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
+ if (PyCallable_Check(py_function)) {
+ RETURN_NOT_OK(VerifyArityAndInput(arity, batch));
+ if (batch[0].is_array()) { // checke 0-th element to select array
callable
+ RETURN_NOT_OK(exec_function_array(batch, py_function, arity.num_args,
out));
+ } else if (batch[0].is_scalar()) { // check 0-th element to select
scalar callable
+ RETURN_NOT_OK(exec_function_scalar(batch, py_function, arity.num_args,
out));
+ } else {
+ return Status::Invalid("Unexpected input type, scalar or array type
expected.");
+ }
+ } else {
+ return Status::ExecutionError("Expected a callable python object.");
+ }
+ return Status::OK();
+ }; // lambda function
+
+ cp::ScalarKernel kernel(
+ cp::KernelSignature::Make(this->input_types(), this->output_type(),
+ this->arity().is_varargs),
+ call_back_lambda);
+ kernel.mem_allocation = this->mem_allocation();
+ kernel.null_handling = this->null_handling();
+ st = func->AddKernel(std::move(kernel));
+ if (!st.ok()) {
+ return Status::ExecutionError("Kernel couldn't be added to the udf : " +
+ st.message());
+ }
Review comment:
Let's just propagate the existing error instead of making another one:
```suggestion
RETURN_NOT_OK(func->AddKernel(std::move(kernel)));
```
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
+ if (PyCallable_Check(py_function)) {
+ RETURN_NOT_OK(VerifyArityAndInput(arity, batch));
+ if (batch[0].is_array()) { // checke 0-th element to select array
callable
+ RETURN_NOT_OK(exec_function_array(batch, py_function, arity.num_args,
out));
+ } else if (batch[0].is_scalar()) { // check 0-th element to select
scalar callable
+ RETURN_NOT_OK(exec_function_scalar(batch, py_function, arity.num_args,
out));
+ } else {
+ return Status::Invalid("Unexpected input type, scalar or array type
expected.");
+ }
+ } else {
+ return Status::ExecutionError("Expected a callable python object.");
+ }
+ return Status::OK();
+ }; // lambda function
+
+ cp::ScalarKernel kernel(
+ cp::KernelSignature::Make(this->input_types(), this->output_type(),
+ this->arity().is_varargs),
+ call_back_lambda);
+ kernel.mem_allocation = this->mem_allocation();
+ kernel.null_handling = this->null_handling();
+ st = func->AddKernel(std::move(kernel));
+ if (!st.ok()) {
+ return Status::ExecutionError("Kernel couldn't be added to the udf : " +
+ st.message());
+ }
+ auto registry = cp::GetFunctionRegistry();
+ st = registry->AddFunction(std::move(func));
Review comment:
Same here.
##########
File path: cpp/src/arrow/python/udf.h
##########
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "arrow/python/platform.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "arrow/compute/api_scalar.h"
+#include "arrow/compute/cast.h"
+#include "arrow/compute/exec.h"
+#include "arrow/compute/function.h"
+#include "arrow/compute/registry.h"
+#include "arrow/datum.h"
+#include "arrow/util/cpu_info.h"
+#include "arrow/util/logging.h"
+
+#include "arrow/python/common.h"
+#include "arrow/python/pyarrow.h"
+#include "arrow/python/visibility.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DECLARE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ ARROW_PYTHON_EXPORT Status exec_function_##FUNCTION_SUFFIX(const
cp::ExecBatch&, \
+ PyObject*, int,
Datum*);
+
+DECLARE_CALL_UDF(Scalar, scalar, scalar)
+DECLARE_CALL_UDF(Array, array, make_array)
+
+#undef DECLARE_CALL_UDF
+
+class ARROW_PYTHON_EXPORT UdfBuilder {
Review comment:
@vibhatha Hmm, I think you misunderstood @westonpace 's suggestion.
AFAIU he was suggesting something like this:
```c++
namespace arrow {
namespace py {
namespace compute {
struct UdfOptions {
std::string func_name;
cp::Function::Kind kind;
cp::Arity arity;
const cp::FunctionDoc func_doc;
std::vector<cp::InputType> in_types;
cp::OutputType out_type;
cp::MemAllocation::type mem_allocation;
cp::NullHandling::type null_handling;
};
Status MakeScalarFunction(PyObject* function, const UdfOptions& options);
```
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
+ Status exec_function_##FUNCTION_SUFFIX(const cp::ExecBatch& batch, PyObject*
function, \
+ int num_args, Datum* out) {
\
+ std::shared_ptr<TYPE_NAME> c_res_data;
\
+ PyObject* arg_tuple = PyTuple_New(num_args);
\
+ for (int arg_id = 0; arg_id < num_args; arg_id++) {
\
+ if (!batch[arg_id].is_##FUNCTION_SUFFIX()) {
\
+ return Status::Invalid("Input type and data type doesn't match");
\
+ }
\
+ auto c_data = batch[arg_id].CONVERT_SUFFIX();
\
+ PyObject* data = wrap_##FUNCTION_SUFFIX(c_data);
\
+ PyTuple_SetItem(arg_tuple, arg_id, data);
\
+ }
\
+ PyObject* result = PyObject_CallObject(function, arg_tuple);
\
+ if (result == NULL) {
\
+ return Status::ExecutionError("Error occured in computation");
\
+ }
\
+ auto res = unwrap_##FUNCTION_SUFFIX(result);
\
+ if (!res.status().ok()) {
\
+ return res.status();
\
+ }
\
+ c_res_data = res.ValueOrDie();
\
+ auto datum = new Datum(c_res_data);
\
+ *out = *datum;
\
+ return Status::OK();
\
+ }
+
+DEFINE_CALL_UDF(Scalar, scalar, scalar)
+DEFINE_CALL_UDF(Array, array, make_array)
+
+#undef DEFINE_CALL_UDF
+
+Status VerifyArityAndInput(cp::Arity arity, const cp::ExecBatch& batch) {
+ bool match = (uint64_t)arity.num_args == batch.values.size();
+ if (!match) {
+ return Status::Invalid(
+ "Function Arity and Input data shape doesn't match, expected {}");
+ }
+ return Status::OK();
+}
+
+Status ScalarUdfBuilder::MakeFunction(PyObject* function, UDFOptions* options)
{
+ Status st;
+ auto doc = this->doc();
+ auto func = std::make_shared<cp::ScalarFunction>(this->name(),
this->arity(), &doc);
+ // creating a copy of objects for the lambda function
+ auto py_function = function;
+ auto arity = this->arity();
+ // lambda function
+ auto call_back_lambda = [py_function, arity](cp::KernelContext* ctx,
+ const cp::ExecBatch& batch,
+ Datum* out) -> Status {
+ PyAcquireGIL lock;
+ if (py_function == NULL) {
+ return Status::ExecutionError("python function cannot be null");
+ }
+ if (PyCallable_Check(py_function)) {
+ RETURN_NOT_OK(VerifyArityAndInput(arity, batch));
+ if (batch[0].is_array()) { // checke 0-th element to select array
callable
+ RETURN_NOT_OK(exec_function_array(batch, py_function, arity.num_args,
out));
+ } else if (batch[0].is_scalar()) { // check 0-th element to select
scalar callable
+ RETURN_NOT_OK(exec_function_scalar(batch, py_function, arity.num_args,
out));
+ } else {
+ return Status::Invalid("Unexpected input type, scalar or array type
expected.");
+ }
+ } else {
+ return Status::ExecutionError("Expected a callable python object.");
Review comment:
Please use `TypeError` here.
##########
File path: python/examples/udf/udf_example.py
##########
@@ -0,0 +1,352 @@
+from typing import List
Review comment:
Can we move the examples to a separate PR and JIRA? It will make this PR
a bit less long to review :-)
##########
File path: cpp/src/arrow/python/udf.cc
##########
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/python/udf.h"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+
+#include "arrow/compute/function.h"
+#include "arrow/python/common.h"
+
+namespace cp = arrow::compute;
+
+namespace arrow {
+
+namespace py {
+
+#define DEFINE_CALL_UDF(TYPE_NAME, FUNCTION_SUFFIX, CONVERT_SUFFIX)
\
Review comment:
Indeed, please let's avoid macros that are more than 2 or 3 lines long.
##########
File path: cpp/examples/arrow/aggregate_example.cc
##########
@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
Review comment:
So this file should be removed, right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
