Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-awkward for openSUSE:Factory checked in at 2023-03-15 18:54:41 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-awkward (Old) and /work/SRC/openSUSE:Factory/.python-awkward.new.31432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-awkward" Wed Mar 15 18:54:41 2023 rev:17 rq:1071882 version:2.1.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-awkward/python-awkward.changes 2023-03-07 16:51:26.625935217 +0100 +++ /work/SRC/openSUSE:Factory/.python-awkward.new.31432/python-awkward.changes 2023-03-15 18:54:54.304510224 +0100 @@ -1,0 +2,6 @@ +Tue Mar 14 22:34:10 UTC 2023 - Dirk Müller <[email protected]> + +- update to 2.1.0: + * bump numpy to 1.17.0 + +------------------------------------------------------------------- Old: ---- awkward-2.0.9.tar.gz New: ---- awkward-2.1.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-awkward.spec ++++++ --- /var/tmp/diff_new_pack.lyPN7x/_old 2023-03-15 18:54:54.824512990 +0100 +++ /var/tmp/diff_new_pack.lyPN7x/_new 2023-03-15 18:54:54.828513012 +0100 @@ -18,7 +18,7 @@ %define awkward_cpp_version 8 Name: python-awkward -Version: 2.0.9 +Version: 2.1.0 Release: 0 Summary: Manipulate arrays of complex data structures as easily as Numpy License: BSD-3-Clause @@ -32,7 +32,7 @@ BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-awkward-cpp = %{awkward_cpp_version} -Requires: python-numpy >= 1.14.5 +Requires: python-numpy >= 1.17.0 Requires: python-packaging Requires: (python-importlib-resources if python-base < 3.9) Requires: (python-typing-extensions >= 4.1.0 if python-base < 3.11) @@ -45,7 +45,7 @@ BuildRequires: %{python_module importlib-resources if %python-base < 3.9} BuildRequires: %{python_module numba >= 0.50 if %python-base < 3.11} BuildRequires: %{python_module numexpr} -BuildRequires: %{python_module numpy >= 1.14.5} +BuildRequires: %{python_module numpy >= 1.17.0} BuildRequires: %{python_module packaging} BuildRequires: %{python_module pandas} BuildRequires: %{python_module pytest-xdist} ++++++ awkward-2.0.9.tar.gz -> awkward-2.1.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/PKG-INFO new/awkward-2.1.0/PKG-INFO --- old/awkward-2.0.9/PKG-INFO 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/PKG-INFO 2023-03-07 20:33:13.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: awkward -Version: 2.0.9 +Version: 2.1.0 Summary: Manipulate JSON-like data with NumPy-like idioms. Project-URL: Bug Tracker, https://github.com/scikit-hep/awkward-1.0/issues Project-URL: Chat, https://gitter.im/Scikit-HEP/awkward-array @@ -36,9 +36,9 @@ Classifier: Topic :: Software Development Classifier: Topic :: Utilities Requires-Python: >=3.7 -Requires-Dist: awkward-cpp==10 +Requires-Dist: awkward-cpp==12 Requires-Dist: importlib-resources; python_version < '3.9' -Requires-Dist: numpy>=1.14.5 +Requires-Dist: numpy>=1.17.0 Requires-Dist: packaging Requires-Dist: typing-extensions>=4.1.0; python_version < '3.11' Description-Content-Type: text/markdown diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/docs/user-guide/how-to-use-header-only-layoutbuilder.md new/awkward-2.1.0/docs/user-guide/how-to-use-header-only-layoutbuilder.md --- old/awkward-2.0.9/docs/user-guide/how-to-use-header-only-layoutbuilder.md 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/docs/user-guide/how-to-use-header-only-layoutbuilder.md 2023-03-07 20:33:13.000000000 +0100 @@ -17,7 +17,8 @@ ```{code-cell} :tags: [hide-cell] -// Make Awkward headers available in this notebook +// Make Awkward headers available in this notebook, because we know these headers are available from the Python sources +// Don't refer to the Git repo location, because they do not exist in sdist #pragma cling add_include_path("../../src/awkward/_connect/header-only") ``` @@ -47,22 +48,27 @@ 2. GrowableBuffer.h 3. LayoutBuilder.h 4. utils.h + +If you are using the CMake project generator, then the `awkward-headers` library can be installed using `FetchContent` for a particular version: +```cmake +include(FetchContent) -It is recommended to download these headers from the [release artifacts on GitHub](https://github.com/scikit-hep/awkward/releases) +set(AWKWARD_VERSION "v2.1.0") -Awkward Array can be installed from PyPI using pip: - -```shell -pip install awkward +FetchContent_Declare( + awkward-headers + URL https://github.com/scikit-hep/awkward/releases/download/${AWKWARD_VERSION}/header-only.zip +) +FetchContent_MakeAvailable(awkward-headers) ``` -To get the `-I` compiler flags needed to pick up the LayoutBuilder from this installation: - -```shell -python -m awkward.config --cflags +The loaded targets can then be linked against, e.g. to link `my_application` against the layout-builder target library: +```cmake +target_link_libraries(my_application awkward::layout-builder) ``` -A user would need to pass these options to the compiler in order to use it. +If you are using a different generator, it is recommended to download these headers from the [release artifacts on GitHub](https://github.com/scikit-hep/awkward/releases). + Three phases of using Layout Builder ------------------------------------ @@ -229,12 +235,58 @@ Passing from C++ to Python -------------------------- -We want NumPy to own the array buffers, so that they get deleted when the Awkward Array goes out of Python scope, not when the LayoutBuilder goes out of C++ scope. For the hand-off, you can allocate memory for those buffers in Python, presumably with `np.empty(nbytes, dtype=np.uint8)` and get void* pointers to these buffers by casting the output of `numpy_array.ctypes.data` (pointer as integer). +We want NumPy to own the array buffers, so that they get deleted when the Awkward Array goes out of Python scope, not when the LayoutBuilder goes out of C++ scope. For the hand-off, one can allocate memory for those buffers in Python, presumably with `np.empty(nbytes, dtype=np.uint8)` and get `void*` pointers to these buffers by casting the output of `numpy_array.ctypes.data` (pointer as integer). Then we can pass everything over the border from C++ to Python using e.g. `pybind11`'s `py::buffer_protocol` for the buffers. + +Alternatively, the Python _capsule_ system can be used to tie the lifetime of the allocated buffers to the calling Python scope. `pybind11` makes this fairly trivial, and also permits us to invoke Python code _from_ C++. We can use this approach to call `ak.from_buffers` in order to build an `ak.Array`: +```cpp +template <typename T> +py::object snapshot_builder(const T& builder) +{ + // How much memory to allocate? + std::map<std::string, size_t> names_nbytes = {}; + builder.buffer_nbytes(names_nbytes); + + // Allocate memory + std::map<std::string, void*> buffers = {}; + for(auto it : names_nbytes) { + uint8_t* ptr = new uint8_t[it.second]; + buffers[it.first] = (void*)ptr; + } + + // Write non-contiguous contents to memory + builder.to_buffers(buffers); + auto from_buffers = py::module::import("awkward").attr("from_buffers"); + + // Build Python dictionary containing arrays + // dtypes not important here as long as they match the underlying buffer + // as Awkward Array calls `frombuffer` to convert to the correct type + py::dict container; + for (auto it: buffers) { + + // Create capsule that frees the allocated data when out of scope + py::capsule free_when_done(it.second, [](void *data) { + uint8_t* dataPtr = reinterpret_cast<uint8_t*>(data); + delete[] dataPtr; + }); + + // Adopt the memory filled by `to_buffers` as a NumPy array + // We only need to return a "buffer" here, but py::array_t let's + // us associate a capsule for destruction, which means that + // Python can own this memory. Therefore, we use py::array_t + uint8_t* data = reinterpret_cast<uint8_t*>(it.second); + container[py::str(it.first)] = py::array_t<uint8_t>( + {names_nbytes[it.first]}, + {sizeof(uint8_t)}, + data, + free_when_done + ); + } + return from_buffers(builder.form(), builder.length(), container); + +} +``` -Now you can pass everything over the border from C++ to Python using pybind11's `py::buffer_protocol` for the buffers, as well as an integer for the length and a string for the Form. More Examples ------------- -Examples for other Layout Builders can be found [here](https://github.com/scikit-hep/awkward/blob/main/tests-cpp/test_1494-layout-builder.cpp). - -<!-- this is just a comment, please remove --> +Examples for other LayoutBuilders can be found [here](https://github.com/scikit-hep/awkward/blob/main/header-only/tests/test_1494-layout-builder.cpp). diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/pyproject.toml new/awkward-2.1.0/pyproject.toml --- old/awkward-2.0.9/pyproject.toml 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/pyproject.toml 2023-03-07 20:33:13.000000000 +0100 @@ -7,7 +7,7 @@ [project] name = "awkward" -version = "2.0.9" +version = "2.1.0" description = "Manipulate JSON-like data with NumPy-like idioms." license = { text = "BSD-3-Clause" } requires-python = ">=3.7" @@ -40,9 +40,9 @@ "Topic :: Utilities", ] dependencies = [ - "awkward_cpp==10", + "awkward_cpp==12", "importlib_resources;python_version < \"3.9\"", - "numpy>=1.14.5", + "numpy>=1.17.0", "packaging", "typing_extensions>=4.1.0; python_version < \"3.11\"" ] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_connect/numpy.py new/awkward-2.1.0/src/awkward/_connect/numpy.py --- old/awkward-2.0.9/src/awkward/_connect/numpy.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_connect/numpy.py 2023-03-07 20:33:13.000000000 +0100 @@ -39,7 +39,7 @@ backend = ak._backends.backend_of(arg, default=None) # We have some array-like object that our backend mechanism understands if backend is not None: - return backend.nplike.to_rectilinear(arg) + return ak.operations.to_numpy(arg) elif isinstance(arg, tuple): return tuple(_to_rectilinear(x) for x in arg) elif isinstance(arg, list): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_nplikes/cupy.py new/awkward-2.1.0/src/awkward/_nplikes/cupy.py --- old/awkward-2.0.9/src/awkward/_nplikes/cupy.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_nplikes/cupy.py 2023-03-07 20:33:13.000000000 +0100 @@ -12,9 +12,6 @@ class Cupy(ArrayModuleNumpyLike): is_eager: Final = False - def to_rectilinear(self, array, *args, **kwargs): - return ak.operations.ak_to_cupy.to_cupy(array, *args, **kwargs) - def __init__(self): import awkward._connect.cuda # noqa: F401 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_nplikes/jax.py new/awkward-2.1.0/src/awkward/_nplikes/jax.py --- old/awkward-2.0.9/src/awkward/_nplikes/jax.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_nplikes/jax.py 2023-03-07 20:33:13.000000000 +0100 @@ -12,9 +12,6 @@ class Jax(ArrayModuleNumpyLike): is_eager: Final = True - def to_rectilinear(self, array, *args, **kwargs): - return ak.operations.ak_to_jax.to_jax(array, *args, **kwargs) - def __init__(self): jax = ak.jax.import_jax() self._module = jax.numpy diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_nplikes/numpy.py new/awkward-2.1.0/src/awkward/_nplikes/numpy.py --- old/awkward-2.0.9/src/awkward/_nplikes/numpy.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_nplikes/numpy.py 2023-03-07 20:33:13.000000000 +0100 @@ -14,9 +14,6 @@ class Numpy(ArrayModuleNumpyLike): is_eager: Final = True - def to_rectilinear(self, array, *args, **kwargs): - return ak.operations.ak_to_numpy.to_numpy(array, *args, **kwargs) - def __init__(self): self._module = numpy diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_nplikes/numpylike.py new/awkward-2.1.0/src/awkward/_nplikes/numpylike.py --- old/awkward-2.0.9/src/awkward/_nplikes/numpylike.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_nplikes/numpylike.py 2023-03-07 20:33:13.000000000 +0100 @@ -614,10 +614,6 @@ def is_c_contiguous(self, x: ArrayLike) -> bool: ... - @abstractmethod - def to_rectilinear(self, array: ArrayLike) -> ArrayLike: - ... - @classmethod @abstractmethod def is_own_array(cls, obj) -> bool: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/_nplikes/typetracer.py new/awkward-2.1.0/src/awkward/_nplikes/typetracer.py --- old/awkward-2.0.9/src/awkward/_nplikes/typetracer.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/_nplikes/typetracer.py 2023-03-07 20:33:13.000000000 +0100 @@ -232,6 +232,10 @@ if not isinstance(shape, tuple): raise wrap_error(TypeError("typetracer shape must be a tuple")) + if any(is_unknown_scalar(x) for x in shape): + raise wrap_error( + TypeError("typetracer shape must be integers or unknown-length") + ) self._shape = shape self._dtype = np.dtype(dtype) @@ -266,10 +270,7 @@ def size(self) -> ShapeItem: size = 1 for item in self._shape: - if ak._util.is_integer(item): - size *= item - else: - return unknown_length + size *= item return size @property @@ -589,9 +590,10 @@ else: raise wrap_error(TypeError) - def to_rectilinear(self, array, *args, **kwargs): - try_touch_shape(array) - raise ak._errors.wrap_error(NotImplementedError) + def _axis_is_valid(self, axis: int, ndim: int) -> bool: + if axis < 0: + axis = axis + ndim + return 0 <= axis < ndim @property def ma(self): @@ -1034,7 +1036,11 @@ maybe_out: ArrayLike | None = None, ) -> TypeTracerArray: try_touch_data(x) - raise ak._errors.wrap_error(NotImplementedError) + if axis is None: + return TypeTracerArray._new(x.dtype, (x.size,)) + else: + assert self._axis_is_valid(axis, x.ndim) + return TypeTracerArray._new(x.dtype, x.shape) def nonzero(self, x: ArrayLike) -> tuple[TypeTracerArray, ...]: # array diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/contents/unionarray.py new/awkward-2.1.0/src/awkward/contents/unionarray.py --- old/awkward-2.0.9/src/awkward/contents/unionarray.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/contents/unionarray.py 2023-03-07 20:33:13.000000000 +0100 @@ -731,21 +731,25 @@ tags_cls: type[Index] = Index8, index_cls: type[Index] = Index64, ) -> tuple[Index, Index]: - offsets = offsets.to_nplike(backend.index_nplike) - counts = [c.to_nplike(backend.index_nplike) for c in counts] + index_nplike = backend.index_nplike - f_offsets = ak.index.Index64(copy.deepcopy(offsets.data)) - contentlen = f_offsets[f_offsets.length - 1] + offsets = offsets.to_nplike(index_nplike) + counts = [c.to_nplike(index_nplike) for c in counts] - tags = tags_cls.empty(contentlen, nplike=backend.index_nplike) - index = index_cls.empty(contentlen, nplike=backend.index_nplike) + f_offsets = ak.index.Index64(index_nplike.asarray(offsets.data, copy=True)) + contentlen = index_nplike.index_as_shape_item( + f_offsets[index_nplike.shape_item_as_index(f_offsets.length - 1)] + ) + + tags = tags_cls.empty(contentlen, nplike=index_nplike) + index = index_cls.empty(contentlen, nplike=index_nplike) for tag, count in enumerate(counts): assert ( - tags.nplike is backend.index_nplike - and index.nplike is backend.index_nplike - and f_offsets.nplike is backend.index_nplike - and count.nplike is backend.index_nplike + tags.nplike is index_nplike + and index.nplike is index_nplike + and f_offsets.nplike is index_nplike + and count.nplike is index_nplike ) Content._selfless_handle_error( backend[ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/jax.py new/awkward-2.1.0/src/awkward/jax.py --- old/awkward-2.0.9/src/awkward/jax.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/jax.py 2023-03-07 20:33:13.000000000 +0100 @@ -32,7 +32,7 @@ Register Awkward Array node types with JAX's tree mechanism. """ try: - import jax # noqa: TID251, F401 + import jax # noqa: TID251 except ModuleNotFoundError: raise wrap_error( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/operations/ak_concatenate.py new/awkward-2.1.0/src/awkward/operations/ak_concatenate.py --- old/awkward-2.0.9/src/awkward/operations/ak_concatenate.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/operations/ak_concatenate.py 2023-03-07 20:33:13.000000000 +0100 @@ -227,7 +227,9 @@ ) ) - counts = backend.index_nplike.zeros(len(nextinputs[0]), dtype=np.int64) + counts = backend.index_nplike.zeros( + nextinputs[0].length, dtype=np.int64 + ) all_counts = [] all_flatten = [] @@ -240,7 +242,7 @@ all_flatten.append(f) offsets = backend.index_nplike.empty( - len(nextinputs[0]) + 1, dtype=np.int64 + nextinputs[0].length + 1, dtype=np.int64 ) offsets[0] = 0 backend.index_nplike.cumsum(counts, maybe_out=offsets[1:]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/src/awkward/operations/ak_unflatten.py new/awkward-2.1.0/src/awkward/operations/ak_unflatten.py --- old/awkward-2.0.9/src/awkward/operations/ak_unflatten.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/src/awkward/operations/ak_unflatten.py 2023-03-07 20:33:13.000000000 +0100 @@ -100,11 +100,11 @@ if counts.is_option and (counts.content.is_numpy or counts.content.is_unknown): mask = counts.mask_as_bool(valid_when=False) - counts = backend.nplike.to_rectilinear( - ak.operations.fill_none(counts, 0, axis=-1, highlevel=False) - ) + counts = ak.operations.fill_none( + counts, 0, axis=-1, highlevel=False + ).to_backend_array() elif counts.is_numpy or counts.is_unknown: - counts = backend.nplike.to_rectilinear(counts) + counts = counts.to_backend_array() mask = False else: raise ak._errors.wrap_error( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/awkward-2.0.9/tests/test_1247_numpy_to_rectilinear_ndarray.py new/awkward-2.1.0/tests/test_1247_numpy_to_rectilinear_ndarray.py --- old/awkward-2.0.9/tests/test_1247_numpy_to_rectilinear_ndarray.py 2023-03-01 16:02:45.000000000 +0100 +++ new/awkward-2.1.0/tests/test_1247_numpy_to_rectilinear_ndarray.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,13 +0,0 @@ -# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE - -import numpy as np -import pytest # noqa: F401 - -import awkward as ak - - -def test(): - array = np.array([1, 2, 9, 0]) - nplike = ak._nplikes.nplike_of(array) - ak_array = ak.operations.from_numpy(array) - assert nplike.to_rectilinear(array).tolist() == ak_array.to_list()
