Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-uproot for openSUSE:Factory checked in at 2023-07-04 15:22:16 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-uproot (Old) and /work/SRC/openSUSE:Factory/.python-uproot.new.23466 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-uproot" Tue Jul 4 15:22:16 2023 rev:14 rq:1096598 version:5.0.9 Changes: -------- --- /work/SRC/openSUSE:Factory/python-uproot/python-uproot.changes 2023-05-30 22:02:20.646975536 +0200 +++ /work/SRC/openSUSE:Factory/.python-uproot.new.23466/python-uproot.changes 2023-07-04 15:23:04.586456770 +0200 @@ -1,0 +2,10 @@ +Mon Jul 3 13:47:02 UTC 2023 - Atri Bhattacharya <badshah...@gmail.com> + +- Update to version 5.0.9: + * fix: if using form remapping start off with full list of + remapped columns [gh#scikit-hep/uproot5#905]. +- Now BuildRequires scikit-hep-testdata >= 0.4.31 for tests. +- For checks, some tests now require a writable tmp dir; create + one in working dir using mktemp and set PYTEST_DEBUG_TEMPROOT. + +------------------------------------------------------------------- Old: ---- uproot-5.0.7.tar.gz New: ---- uproot-5.0.9.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-uproot.spec ++++++ --- /var/tmp/diff_new_pack.GWwQph/_old 2023-07-04 15:23:05.378461530 +0200 +++ /var/tmp/diff_new_pack.GWwQph/_new 2023-07-04 15:23:05.382461554 +0200 @@ -19,7 +19,7 @@ %{?sle15_python_module_pythons} %global modname uproot Name: python-uproot -Version: 5.0.7 +Version: 5.0.9 Release: 0 Summary: ROOT I/O in pure Python and Numpy License: BSD-3-Clause @@ -54,7 +54,7 @@ BuildRequires: %{python_module pytest-timeout} BuildRequires: %{python_module pytest} BuildRequires: %{python_module requests} -BuildRequires: %{python_module scikit-hep-testdata >= 0.4.30} +BuildRequires: %{python_module scikit-hep-testdata >= 0.4.31} BuildRequires: %{python_module xxhash} BuildRequires: %{python_module boost-histogram >= 0.13 if (%python-base without python2-base)} # /SECTION @@ -84,6 +84,7 @@ # pandas tests assume 64bit types skiptests32=("-k" "not (test_jagged_pandas or test_pandas_vector_TLorentzVector or test_iterate_pandas_2 or test_function_iterate_pandas_2 or test_0430)") fi +export PYTEST_DEBUG_TEMPROOT=$(mktemp -d -p ./) %pytest -rfEs -m "not network" "${skiptests32[@]}" %files %{python_files} ++++++ uproot-5.0.7.tar.gz -> uproot-5.0.9.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/.github/workflows/deploy.yml new/uproot-5.0.9/.github/workflows/deploy.yml --- old/uproot-5.0.7/.github/workflows/deploy.yml 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/.github/workflows/deploy.yml 2020-02-02 01:00:00.000000000 +0100 @@ -34,6 +34,6 @@ name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.5 + - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/.pre-commit-config.yaml new/uproot-5.0.9/.pre-commit-config.yaml --- old/uproot-5.0.7/.pre-commit-config.yaml 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/.pre-commit-config.yaml 2020-02-02 01:00:00.000000000 +0100 @@ -1,3 +1,7 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 @@ -19,7 +23,7 @@ - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.0.261" + rev: "v0.0.270" hooks: - id: ruff args: ["--fix", "--show-fixes"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/PKG-INFO new/uproot-5.0.9/PKG-INFO --- old/uproot-5.0.7/PKG-INFO 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/PKG-INFO 2020-02-02 01:00:00.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: uproot -Version: 5.0.7 +Version: 5.0.9 Summary: ROOT I/O in pure Python and NumPy. Project-URL: Download, https://github.com/scikit-hep/uproot5/releases Project-URL: Homepage, https://github.com/scikit-hep/uproot5 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/_dask.py new/uproot-5.0.9/src/uproot/_dask.py --- old/uproot-5.0.7/src/uproot/_dask.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/_dask.py 2020-02-02 01:00:00.000000000 +0100 @@ -47,16 +47,16 @@ with slashes (``/``); otherwise, use the descendant's name as the field name. step_size (int or str): If an integer, the maximum number of entries to - include in each chunk; if a string, the maximum memory_size to include - in each chunk. The string must be a number followed by a memory unit, + include in each chunk/partition; if a string, the maximum memory_size to include + in each chunk/partition. The string must be a number followed by a memory unit, such as "100 MB". Mutually incompatible with steps_per_file: only set step_size or steps_per_file, not both. Cannot be used with ``open_files=False``. steps_per_file (int, default 1): - Subdivide files into the specified number of chunks. Mutually incompatible + Subdivide files into the specified number of chunks/partitions. Mutually incompatible with step_size: only set step_size or steps_per_file, not both. If both ``step_size`` and ``steps_per_file`` are unset, - ``steps_per_file``'s default value of 1 (whole file per chunk) is used, + ``steps_per_file``'s default value of 1 (whole file per chunk/partition) is used, regardless of ``open_files``. library (str or :doc:`uproot.interpretation.library.Library`): The library that is used to represent arrays. If ``library='np'`` it returns a dict @@ -122,6 +122,11 @@ Examples: ``Path("rel/*.root")``, ``"/abs/*.root:tdirectory/ttree"`` * dict: keys are filesystem paths, values are objects-within-ROOT paths. Example: ``{{"/data_v1/*.root": "ttree_v1", "/data_v2/*.root": "ttree_v2"}}`` + * dict: keys are filesystem paths, values are dicts containing objects-within-ROOT and + steps (chunks/partitions) as a list of starts and stops or steps as a list of offsets + Example: ``{{"/data_v1/tree1.root": {"object_path": "ttree_v1", "steps": [[0, 10000], [15000, 20000], ...]}, + "/data_v1/tree2.root": {"object_path": "ttree_v1", "steps": [0, 10000, 20000, ...]}}}`` + (This ``files`` pattern is incompatible with ``step_size`` and ``steps_per_file``.) * already-open TTree objects. * iterables of the above. @@ -149,7 +154,23 @@ array from ``TTrees``. """ - files = uproot._util.regularize_files(files) + files = uproot._util.regularize_files(files, steps_allowed=True) + + is_3arg = [len(x) == 3 for x in files] + if any(is_3arg): + if not all(is_3arg): + raise TypeError( + "partition sizes for some but not all 'files' have been assigned" + ) + if step_size is not unset: + raise TypeError( + "partition sizes for 'files' is incompatible with 'step_size'" + ) + if steps_per_file is not unset: + raise TypeError( + "partition sizes for 'files' is incompatible with 'steps_per_file'" + ) + library = uproot.interpretation.library._regularize_library(library) if step_size is not unset and steps_per_file is not unset: @@ -424,8 +445,14 @@ self.key = key self.interp_options = interp_options - def __call__(self, file_path_object_path_istep_nsteps): - file_path, object_path, istep, nsteps = file_path_object_path_istep_nsteps + def __call__(self, file_path_object_path_istep_nsteps_ischunk): + ( + file_path, + object_path, + istep_or_start, + nsteps_or_stop, + ischunk, + ) = file_path_object_path_istep_nsteps_ischunk ttree = uproot._util.regularize_object_path( file_path, object_path, @@ -434,10 +461,24 @@ self.real_options, ) num_entries = ttree.num_entries - events_per_steps = math.ceil(num_entries / nsteps) - start, stop = (istep * events_per_steps), min( - (istep + 1) * events_per_steps, num_entries - ) + start, stop = istep_or_start, nsteps_or_stop + if not ischunk: + events_per_steps = math.ceil(num_entries / nsteps_or_stop) + start, stop = (istep_or_start * events_per_steps), min( + (istep_or_start + 1) * events_per_steps, num_entries + ) + elif (not 0 <= start < num_entries) or (not 0 <= stop <= num_entries): + raise ValueError( + f"""explicit entry start ({start}) or stop ({stop}) from uproot.dask 'files' argument is out of bounds for file + + {ttree.file.file_path} + +TTree in path + + {ttree.object_path} + +which has {num_entries} entries""" + ) return ttree[self.key].array( library="np", @@ -462,11 +503,14 @@ steps_per_file, ): ttrees = [] + explicit_chunks = [] common_keys = None is_self = [] count = 0 - for file_path, object_path in files: + for file_object_maybechunks in files: + file_path, object_path = file_object_maybechunks[0:2] + obj = uproot._util.regularize_object_path( file_path, object_path, custom_classes, allow_missing, real_options ) @@ -487,6 +531,10 @@ real_filter_branch = filter_branch ttrees.append(obj) + if len(file_object_maybechunks) == 3: + explicit_chunks.append(file_object_maybechunks[2]) + else: + explicit_chunks = None # they all have it or none of them have it new_keys = obj.keys( recursive=recursive, @@ -573,14 +621,31 @@ entry_start = 0 entry_stop = ttree.num_entries - def foreach(start): - stop = min(start + entry_step, entry_stop) # noqa: B023 - length = stop - start - chunks.append(length) # noqa: B023 - chunk_args.append((i, start, stop)) # noqa: B023 - - for start in range(entry_start, entry_stop, entry_step): - foreach(start) + if explicit_chunks is None: + for start in range(entry_start, entry_stop, entry_step): + stop = min(start + entry_step, entry_stop) + length = stop - start + if length > 0: + chunks.append(length) + chunk_args.append((i, start, stop)) + else: + for start, stop in explicit_chunks[i]: + if (not 0 <= start < entry_stop) or (not 0 <= stop <= entry_stop): + raise ValueError( + f"""explicit entry start ({start}) or stop ({stop}) from uproot.dask 'files' argument is out of bounds for file + + {ttree.file.file_path} + +TTree in path + + {ttree.object_path} + +which has {entry_stop} entries""" + ) + length = stop - start + if length > 0: + chunks.append(length) + chunk_args.append((i, start, stop)) if len(chunk_args) == 0: chunks.append(0) @@ -610,7 +675,7 @@ interp_options, steps_per_file, ): - ffile_path, fobject_path = files[0] + ffile_path, fobject_path = files[0][0:2] obj = uproot._util.regularize_object_path( ffile_path, fobject_path, custom_classes, allow_missing, real_options ) @@ -631,24 +696,46 @@ else: dt, inner_shape = dt.subdtype + partitions = [] partition_args = [] - for ifile_path, iobject_path in files: - for istep in range(steps_per_file): - partition_args.append( - ( - ifile_path, - iobject_path, - istep, - steps_per_file, + for ifile_iobject_maybeichunks in files: + ifile_path, iobject_path = ifile_iobject_maybeichunks[0:2] + + chunks = None + if len(ifile_iobject_maybeichunks) == 3: + chunks = ifile_iobject_maybeichunks[2] + + if chunks is not None: + partitions.extend([stop - start for start, stop in chunks]) + for start, stop in chunks: + partition_args.append( + ( + ifile_path, + iobject_path, + start, + stop, + True, + ) + ) + else: + partitions.extend([numpy.nan] * steps_per_file) + for istep in range(steps_per_file): + partition_args.append( + ( + ifile_path, + iobject_path, + istep, + steps_per_file, + False, + ) ) - ) dask_dict[key] = _dask_array_from_map( _UprootOpenAndReadNumpy( custom_classes, allow_missing, real_options, key, interp_options ), partition_args, - chunks=((numpy.nan,) * len(files) * steps_per_file,), + chunks=(tuple(partitions),), dtype=dt, label=f"{key}-from-uproot", ) @@ -678,7 +765,10 @@ if self.form_mapping is not None: awkward = uproot.extras.awkward() - actual_form = self.rendered_form.select_columns(self.common_keys) + if set(self.common_keys) != set(self.rendered_form.columns()): + actual_form = self.rendered_form.select_columns(self.common_keys) + else: + actual_form = self.rendered_form mapping, buffer_key = self.form_mapping.create_column_mapping_and_key( self.ttrees[i], start, stop, self.interp_options @@ -755,8 +845,14 @@ self.form_mapping = form_mapping self.rendered_form = rendered_form - def __call__(self, file_path_object_path_istep_nsteps): - file_path, object_path, istep, nsteps = file_path_object_path_istep_nsteps + def __call__(self, file_path_object_path_istep_nsteps_ischunk): + ( + file_path, + object_path, + istep_or_start, + nsteps_or_stop, + ischunk, + ) = file_path_object_path_istep_nsteps_ischunk ttree = uproot._util.regularize_object_path( file_path, object_path, @@ -765,15 +861,32 @@ self.real_options, ) num_entries = ttree.num_entries - events_per_step = math.ceil(num_entries / nsteps) - start, stop = (istep * events_per_step), min( - (istep + 1) * events_per_step, num_entries - ) + start, stop = istep_or_start, nsteps_or_stop + if not ischunk: + events_per_step = math.ceil(num_entries / nsteps_or_stop) + start, stop = (istep_or_start * events_per_step), min( + (istep_or_start + 1) * events_per_step, num_entries + ) + elif (not 0 <= start < num_entries) or (not 0 <= stop <= num_entries): + raise ValueError( + f"""explicit entry start ({start}) or stop ({stop}) from uproot.dask 'files' argument is out of bounds for file + + {ttree.file.file_path} + +TTree in path + + {ttree.object_path} + +which has {num_entries} entries""" + ) if self.form_mapping is not None: awkward = uproot.extras.awkward() - actual_form = self.rendered_form.select_columns(self.common_keys) + if set(self.common_keys) != set(self.rendered_form.columns()): + actual_form = self.rendered_form.select_columns(self.common_keys) + else: + actual_form = self.rendered_form mapping, buffer_key = self.form_mapping.create_column_mapping_and_key( ttree, start, stop, self.interp_options @@ -855,8 +968,9 @@ if form_mapping is not None: form = form_mapping(form) - empty_arr = form.length_zero_array( - behavior=None if form_mapping is None else form_mapping.behavior + empty_arr = awkward.Array( + form.length_zero_array(highlevel=False), + behavior=None if form_mapping is None else form_mapping.behavior, ) return dask_awkward.core.typetracer_array(empty_arr), form @@ -881,11 +995,14 @@ awkward = uproot.extras.awkward() ttrees = [] + explicit_chunks = [] common_keys = None is_self = [] count = 0 - for file_path, object_path in files: + for file_object_maybechunks in files: + file_path, object_path = file_object_maybechunks[0:2] + obj = uproot._util.regularize_object_path( file_path, object_path, custom_classes, allow_missing, real_options ) @@ -906,6 +1023,10 @@ real_filter_branch = filter_branch ttrees.append(obj) + if len(file_object_maybechunks) == 3: + explicit_chunks.append(file_object_maybechunks[2]) + else: + explicit_chunks = None # they all have it or none of them have it new_keys = obj.keys( recursive=recursive, @@ -976,17 +1097,37 @@ entry_step = int(round(step_sum / len(ttrees))) + divisions = [0] partition_args = [] for i, ttree in enumerate(ttrees): entry_start = 0 entry_stop = ttree.num_entries - def foreach(start): - stop = min(start + entry_step, entry_stop) # noqa: B023 - partition_args.append((i, start, stop)) # noqa: B023 + if explicit_chunks is None: + for start in range(entry_start, entry_stop, entry_step): + stop = min(start + entry_step, entry_stop) + length = stop - start + if length > 0: + divisions.append(divisions[-1] + length) + partition_args.append((i, start, stop)) + else: + for start, stop in explicit_chunks[i]: + if (not 0 <= start < entry_stop) or (not 0 <= stop <= entry_stop): + raise ValueError( + f"""explicit entry start ({start}) or stop ({stop}) from uproot.dask 'files' argument is out of bounds for file + + {ttree.file.file_path} - for start in range(entry_start, entry_stop, entry_step): - foreach(start) +TTree in path + + {ttree.object_path} + +which has {entry_stop} entries""" + ) + length = stop - start + if length > 0: + divisions.append(divisions[-1] + length) + partition_args.append((i, start, stop)) meta, form = _get_meta_array( awkward, @@ -998,17 +1139,20 @@ ) if len(partition_args) == 0: + divisions.append(0) partition_args.append((0, 0, 0)) + return dask_awkward.from_map( _UprootRead( ttrees, - common_keys, + common_keys if form_mapping is None else form.columns(), common_keys, interp_options, form_mapping=form_mapping, rendered_form=None if form_mapping is None else form, ), partition_args, + divisions=tuple(divisions), label="from-uproot", behavior=None if form_mapping is None else form_mapping.behavior, meta=meta, @@ -1032,7 +1176,8 @@ dask_awkward = uproot.extras.dask_awkward() awkward = uproot.extras.awkward() - ffile_path, fobject_path = files[0] + ffile_path, fobject_path = files[0][0:2] + obj = uproot._util.regularize_object_path( ffile_path, fobject_path, custom_classes, allow_missing, real_options ) @@ -1053,30 +1198,52 @@ interp_options.get("ak_add_doc"), ) + divisions = [0] partition_args = [] - for ifile_path, iobject_path in files: - for istep in range(steps_per_file): - partition_args.append( - ( - ifile_path, - iobject_path, - istep, - steps_per_file, + for ifile_iobject_maybeichunks in files: + chunks = None + ifile_path, iobject_path = ifile_iobject_maybeichunks[0:2] + if len(ifile_iobject_maybeichunks) == 3: + chunks = ifile_iobject_maybeichunks[2] + + if chunks is not None: + for start, stop in chunks: + divisions.append(divisions[-1] + (stop - start)) + partition_args.append( + ( + ifile_path, + iobject_path, + start, + stop, + True, + ) + ) + else: + divisions = None # they all have it or none of them have it + for istep in range(steps_per_file): + partition_args.append( + ( + ifile_path, + iobject_path, + istep, + steps_per_file, + False, + ) ) - ) return dask_awkward.from_map( _UprootOpenAndRead( custom_classes, allow_missing, real_options, - common_keys, + common_keys if form_mapping is None else form.columns(), common_keys, interp_options, form_mapping=form_mapping, rendered_form=None if form_mapping is None else form, ), partition_args, + divisions=None if divisions is None else tuple(divisions), label="from-uproot", behavior=None if form_mapping is None else form_mapping.behavior, meta=meta, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/_util.py new/uproot-5.0.9/src/uproot/_util.py --- old/uproot-5.0.7/src/uproot/_util.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/_util.py 2020-02-02 01:00:00.000000000 +0100 @@ -808,9 +808,42 @@ _regularize_files_isglob = re.compile(r"[\*\?\[\]{}]") -def _regularize_files_inner(files, parse_colon, counter, HasBranches): +def regularize_steps(steps): + out = numpy.array(steps) + + if isinstance(steps, dict) or not issubclass(out.dtype.type, numpy.integer): + raise TypeError( + "'files' argument's steps must be an iterable of integer offsets or start-stop pairs." + ) + + if len(out.shape) == 1: + if len(out) == 0 or not numpy.all(out[1:] >= out[:-1]): + raise ValueError( + "if 'files' argument's steps are (one-dimensional) offsets, they must be non-empty and monotonically increasing" + ) + + elif len(out.shape) == 2: + if not (out.shape[1] == 2 and all(out[:, 1] >= out[:, 0])): + raise ValueError( + "if 'files' argument's steps are (two-dimensional) start-stop pairs, all stops must be greater than or equal to their corresponding starts" + ) + + else: + raise TypeError( + "'files' argument's steps must be an iterable of integer offsets or a list of pairs of integer starts and stops." + ) + + if len(out.shape) == 1: + out = numpy.stack((out[:-1], out[1:]), axis=1) + + return out.tolist() + + +def _regularize_files_inner(files, parse_colon, counter, HasBranches, steps_allowed): files2 = regularize_path(files) + maybe_steps = None + if isstr(files2) and not isstr(files): parse_colon = False files = files2 @@ -824,12 +857,12 @@ parsed_url = urlparse(file_path) if parsed_url.scheme.upper() in _remote_schemes: - yield file_path, object_path + yield file_path, object_path, maybe_steps else: expanded = os.path.expanduser(file_path) if _regularize_files_isglob.search(expanded) is None: - yield file_path, object_path + yield file_path, object_path, maybe_steps else: matches = list(_regularize_files_braces.finditer(expanded)) @@ -849,26 +882,43 @@ for result in results: for match in glob.glob(result): if match not in seen: - yield match, object_path + yield match, object_path, maybe_steps seen.add(match) elif isinstance(files, HasBranches): - yield files, None + yield files, None, maybe_steps elif isinstance(files, dict): - for key, object_path in files.items(): - for file_path, _ in _regularize_files_inner( - key, False, counter, HasBranches + for key, maybe_object_path in files.items(): + if not isinstance(maybe_object_path, (type(None), str, dict)): + raise TypeError("object_path may only be a string, dict, or None") + if isinstance(maybe_object_path, dict): + maybe_steps = maybe_object_path.get("steps", None) + object_path = maybe_object_path.get("object_path", None) + if maybe_steps is not None: + if not steps_allowed: + raise TypeError( + "unrecognized 'files' pattern for this function ('steps' are only allowed in uproot.dask)" + ) + maybe_steps = regularize_steps(maybe_steps) + else: + object_path = maybe_object_path + for file_path, _, _ in _regularize_files_inner( + key, + False, + counter, + HasBranches, + steps_allowed, ): - yield file_path, object_path + yield file_path, object_path, maybe_steps elif isinstance(files, Iterable): for file in files: counter[0] += 1 - for file_path, object_path in _regularize_files_inner( - file, parse_colon, counter, HasBranches + for file_path, object_path, maybe_steps in _regularize_files_inner( + file, parse_colon, counter, HasBranches, steps_allowed ): - yield file_path, object_path + yield file_path, object_path, maybe_steps else: raise TypeError( @@ -879,7 +929,7 @@ ) -def regularize_files(files): +def regularize_files(files, steps_allowed): """ Common code for regularizing the possible file inputs accepted by uproot so they can be used by uproot internal functions. """ @@ -888,16 +938,21 @@ out = [] seen = set() counter = [0] - for file_path, object_path in _regularize_files_inner( - files, True, counter, HasBranches + for file_path, object_path, maybe_steps in _regularize_files_inner( + files, True, counter, HasBranches, steps_allowed ): if isstr(file_path): key = (counter[0], file_path, object_path) if key not in seen: out.append((file_path, object_path)) + if maybe_steps is not None: + out[-1] = (*out[-1], maybe_steps) + seen.add(key) else: out.append((file_path, object_path)) + if maybe_steps is not None: + out[-1] = (*out[-1], maybe_steps) if len(out) == 0: raise _file_not_found(files) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/behaviors/TAxis.py new/uproot-5.0.9/src/uproot/behaviors/TAxis.py --- old/uproot-5.0.7/src/uproot/behaviors/TAxis.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/behaviors/TAxis.py 2020-02-02 01:00:00.000000000 +0100 @@ -183,7 +183,7 @@ if fLabels is not None and len(fLabels) == fNbins: out = [str(x) for x in fLabels] if flow: - return ["underflow", *out] + ["overflow"] + return ["underflow", *out, "overflow"] else: return out else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/behaviors/TBranch.py new/uproot-5.0.9/src/uproot/behaviors/TBranch.py --- old/uproot-5.0.7/src/uproot/behaviors/TBranch.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/behaviors/TBranch.py 2020-02-02 01:00:00.000000000 +0100 @@ -175,7 +175,7 @@ array from ``TTrees``. * :doc:`uproot._dask.dask`: returns an unevaluated Dask array from ``TTrees``. """ - files = uproot._util.regularize_files(files) + files = uproot._util.regularize_files(files, steps_allowed=False) decompression_executor, interpretation_executor = _regularize_executors( decompression_executor, interpretation_executor, None ) @@ -343,7 +343,7 @@ single concatenated array from ``TTrees``. * :doc:`uproot._dask.dask`: returns an unevaluated Dask array from ``TTrees``. """ - files = uproot._util.regularize_files(files) + files = uproot._util.regularize_files(files, steps_allowed=False) decompression_executor, interpretation_executor = _regularize_executors( decompression_executor, interpretation_executor, None ) @@ -2490,7 +2490,7 @@ {} -instead, try library="np" instead of library="ak" or globally set uproot.default_library +instead, try library="np" rather than library="ak" or globally set uproot.default_library in file {} in object {}""".format( @@ -3237,7 +3237,7 @@ def __delitem__(self, where): del self.dict[where] - def __iter__(self, where): + def __iter__(self): yield from self.dict def __len__(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/behaviors/TH1.py new/uproot-5.0.9/src/uproot/behaviors/TH1.py --- old/uproot-5.0.7/src/uproot/behaviors/TH1.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/behaviors/TH1.py 2020-02-02 01:00:00.000000000 +0100 @@ -20,8 +20,8 @@ fNbins = axis.member("fNbins") fXbins = axis.member("fXbins", none_if_missing=True) - if axis.member("fLabels") is not None: - fLabels = axis.member("fLabels") + fLabels = axis.member("fLabels", none_if_missing=True) + if fLabels is not None: try: labels = [int(x) for x in fLabels] category_cls = boost_histogram.axis.IntCategory diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/models/TAtt.py new/uproot-5.0.9/src/uproot/models/TAtt.py --- old/uproot-5.0.7/src/uproot/models/TAtt.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/models/TAtt.py 2020-02-02 01:00:00.000000000 +0100 @@ -526,6 +526,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -679,6 +680,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/models/TGraph.py new/uproot-5.0.9/src/uproot/models/TGraph.py --- old/uproot-5.0.7/src/uproot/models/TGraph.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/models/TGraph.py 2020-02-02 01:00:00.000000000 +0100 @@ -249,6 +249,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -455,6 +456,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -666,6 +668,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/models/TH.py new/uproot-5.0.9/src/uproot/models/TH.py --- old/uproot-5.0.7/src/uproot/models/TH.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/models/TH.py 2020-02-02 01:00:00.000000000 +0100 @@ -338,6 +338,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -814,6 +815,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1119,6 +1121,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1328,6 +1331,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1517,6 +1521,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1679,6 +1684,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1836,6 +1842,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -1993,6 +2000,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2155,6 +2163,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2317,6 +2326,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2480,6 +2490,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2638,6 +2649,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2801,6 +2813,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -2964,6 +2977,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3127,6 +3141,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3291,6 +3306,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3450,6 +3466,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3614,6 +3631,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3778,6 +3796,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -3978,6 +3997,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -4217,6 +4237,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: @@ -4458,6 +4479,7 @@ raise uproot.interpretation.objects.CannotBeAwkward( "classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded" ) + context = context.copy() context["breadcrumbs"] = context["breadcrumbs"] + (cls,) contents = {} if context["header"]: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/models/TTree.py new/uproot-5.0.9/src/uproot/models/TTree.py --- old/uproot-5.0.7/src/uproot/models/TTree.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/models/TTree.py 2020-02-02 01:00:00.000000000 +0100 @@ -959,7 +959,7 @@ Returns an iterator over the number of entries over each TTree in the input. This is a shortcut method and reads lesser data than normal file opening. """ - paths2 = uproot._util.regularize_files(paths) + paths2 = uproot._util.regularize_files(paths, steps_allowed=False) if isinstance(paths, dict): paths = list(paths.items()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/reading.py new/uproot-5.0.9/src/uproot/reading.py --- old/uproot-5.0.7/src/uproot/reading.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/reading.py 2020-02-02 01:00:00.000000000 +0100 @@ -1484,7 +1484,7 @@ See :ref:`uproot.reading.ReadOnlyDirectory.path` for the path as a tuple of strings. """ - return "/".join(("", *self._path) + ("",)).replace("//", "/") + return "/".join(("", *self._path, "")).replace("//", "/") @property def file_path(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/streamers.py new/uproot-5.0.9/src/uproot/streamers.py --- old/uproot-5.0.7/src/uproot/streamers.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/streamers.py 2020-02-02 01:00:00.000000000 +0100 @@ -242,6 +242,7 @@ " from awkward.forms import NumpyForm, ListOffsetForm, RegularForm, RecordForm", " if cls in context['breadcrumbs']:", " raise uproot.interpretation.objects.CannotBeAwkward('classes that can contain members of the same type cannot be Awkward Arrays because the depth of instances is unbounded')", + " context = context.copy()", " context['breadcrumbs'] = context['breadcrumbs'] + (cls,)", " contents = {}", " if context['header']:", @@ -553,9 +554,10 @@ self._members["fTypeName"] = _canonical_typename(cursor.string(chunk, context)) - if self._members["fType"] == 11 and self._members["fTypeName"] in ( - "Bool_t" or "bool" - ): + if self._members["fType"] == 11 and self._members["fTypeName"] in { + "Bool_t", + "bool", + }: self._members["fType"] = 18 if self._instance_version <= 2: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/version.py new/uproot-5.0.9/src/uproot/version.py --- old/uproot-5.0.7/src/uproot/version.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/version.py 2020-02-02 01:00:00.000000000 +0100 @@ -12,7 +12,7 @@ import re -__version__ = "5.0.7" +__version__ = "5.0.9" version = __version__ version_info = tuple(re.split(r"[-\.]", __version__)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/writing/_cascadetree.py new/uproot-5.0.9/src/uproot/writing/_cascadetree.py --- old/uproot-5.0.7/src/uproot/writing/_cascadetree.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/writing/_cascadetree.py 2020-02-02 01:00:00.000000000 +0100 @@ -983,8 +983,36 @@ leaf_title_length = (1 if len(leaf_title) < 255 else 5) + len(leaf_title) leaf_header = numpy.array( - [64, 0, 0, 76, 0, 1, 64, 0, 0, 54, 0, 2, 64, 0] - + [0, 30, 0, 1, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0], + [ + 64, + 0, + 0, + 76, + 0, + 1, + 64, + 0, + 0, + 54, + 0, + 2, + 64, + 0, + 0, + 30, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 3, + 0, + 0, + 0, + ], numpy.uint8, ) tmp = leaf_header[0:4].view(">u4") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/src/uproot/writing/writable.py new/uproot-5.0.9/src/uproot/writing/writable.py --- old/uproot-5.0.7/src/uproot/writing/writable.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/src/uproot/writing/writable.py 2020-02-02 01:00:00.000000000 +0100 @@ -534,7 +534,7 @@ Path of directory names to this subdirectory as a single string, delimited by slashes. """ - return "/".join(("", *self._path) + ("",)).replace("//", "/") + return "/".join(("", *self._path, "")).replace("//", "/") @property def file_path(self): @@ -1651,7 +1651,7 @@ Path of directory names to this TTree as a single string, delimited by slashes. """ - return "/".join(("", *self._path) + ("",)).replace("//", "/") + return "/".join(("", *self._path, "")).replace("//", "/") @property def file_path(self): @@ -1994,7 +1994,7 @@ Path of directory names to this RNTuple as a single string, delimited by slashes. """ - return "/".join(("", *self._path) + ("",)).replace("//", "/") + return "/".join(("", *self._path, "")).replace("//", "/") @property def file_path(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/tests/test_0603-dask-delayed-open.py new/uproot-5.0.9/tests/test_0603-dask-delayed-open.py --- old/uproot-5.0.7/tests/test_0603-dask-delayed-open.py 2020-02-02 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/tests/test_0603-dask-delayed-open.py 2020-02-02 01:00:00.000000000 +0100 @@ -38,3 +38,50 @@ == true_val["px1"] ) assert arr.all() + + +@pytest.mark.parametrize("open_files", [False, True]) +@pytest.mark.parametrize("library", ["np", "ak"]) +def test_supplied_steps(open_files, library): + filename1 = skhep_testdata.data_path("uproot-Zmumu.root") + filename2 = skhep_testdata.data_path("uproot-Zmumu-uncompressed.root") + true_val = uproot.concatenate( + [filename1 + ":events", filename2 + ":events"], "px1", library=library + )["px1"] + + files = [filename1, filename2] + daskarr = uproot.dask(files, open_files=open_files, library=library)["px1"] + + if library == "ak": + if open_files: + assert daskarr.divisions == (0, 2304, 4608) + else: + assert daskarr.divisions == (None, None, None) + else: + if open_files: + assert daskarr.chunks == ((2304, 2304),) + else: + assert daskarr.chunks == ((numpy.nan, numpy.nan),) + + assert daskarr.compute().tolist() == true_val.tolist() + + steps1 = [0, 1000, 2304] + steps2 = [[0, 1200], [1200, 2304]] + files = { + filename1: {"object_path": "events", "steps": steps1}, + filename2: {"object_path": "events", "steps": steps2}, + } + daskarr = uproot.dask(files, open_files=open_files, library=library)["px1"] + + if library == "ak": + if open_files: + assert daskarr.divisions == (0, 1000, 2304, 3504, 4608) + else: + assert daskarr.divisions == (0, 1000, 2304, 3504, 4608) + else: + if open_files: + assert daskarr.chunks == ((1000, 1304, 1200, 1104),) + else: + assert daskarr.chunks == ((1000, 1304, 1200, 1104),) + + assert daskarr.compute().tolist() == true_val.tolist() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/uproot-5.0.7/tests/test_0886-fix-awkward-form-breadcrumbs.py new/uproot-5.0.9/tests/test_0886-fix-awkward-form-breadcrumbs.py --- old/uproot-5.0.7/tests/test_0886-fix-awkward-form-breadcrumbs.py 1970-01-01 01:00:00.000000000 +0100 +++ new/uproot-5.0.9/tests/test_0886-fix-awkward-form-breadcrumbs.py 2020-02-02 01:00:00.000000000 +0100 @@ -0,0 +1,10 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE + +import uproot +import skhep_testdata + + +def test_fix_awkward_form_breadcrumbs(): + file = uproot.open(skhep_testdata.data_path("uproot-issue-880.root")) + tree = file["Z"] + assert tree.num_entries == 116