commit: 6d7268c6c8e47b43efd5550b05d3ad587704bdff Author: Alfredo Tupone <tupone <AT> gentoo <DOT> org> AuthorDate: Sun May 7 20:15:13 2023 +0000 Commit: Alfredo Tupone <tupone <AT> gentoo <DOT> org> CommitDate: Sun May 7 20:15:42 2023 +0000 URL: https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=6d7268c6
sci-libs/datasets: new package, add 2.11.0 Signed-off-by: Alfredo Tupone <tupone <AT> gentoo.org> sci-libs/datasets/Manifest | 1 + sci-libs/datasets/datasets-2.11.0.ebuild | 53 +++++++ .../datasets/files/datasets-2.11.0-tests.patch | 168 +++++++++++++++++++++ sci-libs/datasets/metadata.xml | 12 ++ 4 files changed, 234 insertions(+) diff --git a/sci-libs/datasets/Manifest b/sci-libs/datasets/Manifest new file mode 100644 index 000000000000..37284ce3b3e7 --- /dev/null +++ b/sci-libs/datasets/Manifest @@ -0,0 +1 @@ +DIST datasets-2.11.0.gh.tar.gz 2141289 BLAKE2B 0fb471dd6ee5de3831eb6586c4a15e67381262470b72d5ab02ee87dfc7977cb4d40e04da6507049d1e47cb8948cad11988bb7627293b48231e1cd413d2cfb885 SHA512 9ec2274d7978e3dde1b2f8ce78dd65bdf66742bbfee7b8672af46216aeaae3ef5c4604a8a5ea0bdee808f1c362cca9a122c16d2e9a161678148e581e4cd5c863 diff --git a/sci-libs/datasets/datasets-2.11.0.ebuild b/sci-libs/datasets/datasets-2.11.0.ebuild new file mode 100644 index 000000000000..0ff1a4f0b386 --- /dev/null +++ b/sci-libs/datasets/datasets-2.11.0.ebuild @@ -0,0 +1,53 @@ +# Copyright 2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DISTUTILS_USE_PEP517=setuptools +PYTHON_COMPAT=( python3_{9..11} ) +DISTUTILS_SINGLE_IMPL=1 +inherit distutils-r1 + +DESCRIPTION="access and share datasets for Audio, Computer Vision, and NLP tasks" +HOMEPAGE=" + https://pypi.org/project/datasets/ +" +SRC_URI="https://github.com/huggingface/${PN}/archive/refs/tags/${PV}.tar.gz + -> ${P}.gh.tar.gz" +IUSE="test" + +LICENSE="Apache-2.0" +SLOT="0" +KEYWORDS="~amd64" + +RDEPEND=" + ${PYTHON_DEPS} + sci-libs/pytorch[${PYTHON_SINGLE_USEDEP}] + $(python_gen_cond_dep ' + dev-python/pyarrow[${PYTHON_USEDEP},parquet,snappy] + dev-python/fsspec[${PYTHON_USEDEP}] + dev-python/multiprocess[${PYTHON_USEDEP}] + dev-python/aiohttp[${PYTHON_USEDEP}] + dev-python/xxhash[${PYTHON_USEDEP}] + dev-python/zstandard[${PYTHON_USEDEP}] + dev-python/absl-py[${PYTHON_USEDEP}] + sci-libs/scikit-learn[${PYTHON_USEDEP}] + ') +" +DEPEND="${RDEPEND}" +BDEPEND="test? ( + $(python_gen_cond_dep ' + dev-python/pytest-datadir[${PYTHON_USEDEP}] + ') +)" + +PATCHES=( "${FILESDIR}"/${P}-tests.patch ) + +distutils_enable_tests pytest + +src_prepare() { + # Require jiwer + rm metrics/cer/test_cer.py || die + + distutils-r1_src_prepare +} diff --git a/sci-libs/datasets/files/datasets-2.11.0-tests.patch b/sci-libs/datasets/files/datasets-2.11.0-tests.patch new file mode 100644 index 000000000000..01e5d9c70e7b --- /dev/null +++ b/sci-libs/datasets/files/datasets-2.11.0-tests.patch @@ -0,0 +1,168 @@ +--- a/tests/test_metric_common.py 2023-05-04 18:48:48.550861318 +0200 ++++ b/tests/test_metric_common.py 2023-05-04 18:50:25.787364577 +0200 +@@ -93,6 +93,7 @@ + INTENSIVE_CALLS_PATCHER = {} + metric_name = None + ++ @pytest.mark.skip(reason="disabling, depends on bert_score, bleurt, math_equivalence, coval, nltk, faiss, mauve, rouge_score, sacrebleu, sacremoses ...") + def test_load_metric(self, metric_name): + doctest.ELLIPSIS_MARKER = "[...]" + metric_module = importlib.import_module( +--- a/tests/test_hf_gcp.py 2023-05-04 19:33:31.150825303 +0200 ++++ b/tests/test_hf_gcp.py 2023-05-04 19:40:08.401759538 +0200 +@@ -69,6 +69,7 @@ + self.assertTrue(os.path.exists(datset_info_path)) + + ++@pytest.mark.skip(reason="require apache_beam") + @pytest.mark.integration + def test_wikipedia_frr(tmp_path_factory): + tmp_dir = tmp_path_factory.mktemp("test_hf_gcp") / "test_wikipedia_simple" +--- a/tests/test_distributed.py 2023-05-04 19:43:09.861275030 +0200 ++++ b/tests/test_distributed.py 2023-05-04 19:44:17.608326722 +0200 +@@ -55,6 +55,7 @@ + assert len({tuple(x.values()) for ds in datasets_per_rank for x in ds}) == full_size + + ++@pytest.mark.skip(reason="require distributed torch") + @pytest.mark.parametrize("streaming", [False, True]) + @require_torch + @pytest.mark.skipif(os.name == "nt", reason="execute_subprocess_async doesn't support windows") +@@ -76,6 +77,7 @@ + execute_subprocess_async(cmd, env=os.environ.copy()) + + ++@pytest.mark.skip(reason="require distributed torch") + @pytest.mark.parametrize( + "nproc_per_node, num_workers", + [ +--- a/tests/utils.py 2023-05-06 08:43:16.251987543 +0200 ++++ b/tests/utils.py 2023-05-06 08:44:24.467952870 +0200 +@@ -54,8 +54,8 @@ + # Audio + require_sndfile = pytest.mark.skipif( + # On Windows and OS X, soundfile installs sndfile +- find_spec("soundfile") is None or version.parse(importlib_metadata.version("soundfile")) < version.parse("0.12.0"), +- reason="test requires sndfile>=0.12.1: 'pip install \"soundfile>=0.12.1\"'; ", ++ True, ++ reason="test requires librosa", + ) + + # Beam +--- a/tests/features/test_audio.py 2023-05-06 09:03:58.680108142 +0200 ++++ a/tests/features/test_audio.py 2023-05-06 09:05:50.463407967 +0200 +@@ -57,6 +57,7 @@ + assert features.arrow_schema == pa.schema({"sequence_of_audios": pa.list_(Audio().pa_type)}) + + ++@pytest.mark.skip(reason="require librosa") + @pytest.mark.parametrize( + "build_example", + [ +@@ -82,6 +82,7 @@ + assert decoded_example.keys() == {"path", "array", "sampling_rate"} + + ++@pytest.mark.skip(reason="require librosa") + @pytest.mark.parametrize( + "build_example", + [ +@@ -148,6 +149,7 @@ + assert decoded_example["sampling_rate"] == 48000 + + ++@pytest.mark.skip(reason="require librosa") + @pytest.mark.parametrize("sampling_rate", [16_000, 48_000]) + def test_audio_decode_example_pcm(shared_datadir, sampling_rate): + audio_path = str(shared_datadir / "test_audio_16000.pcm") +@@ -416,6 +417,7 @@ + assert column[0]["sampling_rate"] == 16000 + + ++@pytest.mark.skip(reason="require librosa") + @pytest.mark.parametrize( + "build_data", + [ +@@ -440,6 +442,7 @@ + assert item["audio"].keys() == {"path", "array", "sampling_rate"} + + ++@pytest.mark.skip(reason="require librosa") + def test_dataset_concatenate_audio_features(shared_datadir): + # we use a different data structure between 1 and 2 to make sure they are compatible with each other + audio_path = str(shared_datadir / "test_audio_44100.wav") +@@ -453,6 +456,7 @@ + assert concatenated_dataset[1]["audio"]["array"].shape == dset2[0]["audio"]["array"].shape + + ++@pytest.mark.skip(reason="require librosa") + def test_dataset_concatenate_nested_audio_features(shared_datadir): + # we use a different data structure between 1 and 2 to make sure they are compatible with each other + audio_path = str(shared_datadir / "test_audio_44100.wav") +@@ -616,6 +616,7 @@ + assert isinstance(ds, Dataset) + + ++@require_sndfile + def test_dataset_with_audio_feature_undecoded(shared_datadir): + audio_path = str(shared_datadir / "test_audio_44100.wav") + data = {"audio": [audio_path]} +@@ -633,6 +634,7 @@ + assert column[0] == {"path": audio_path, "bytes": None} + + ++@require_sndfile + def test_formatted_dataset_with_audio_feature_undecoded(shared_datadir): + audio_path = str(shared_datadir / "test_audio_44100.wav") + data = {"audio": [audio_path]} +@@ -664,6 +666,7 @@ + assert column[0] == {"path": audio_path, "bytes": None} + + ++@require_sndfile + def test_dataset_with_audio_feature_map_undecoded(shared_datadir): + audio_path = str(shared_datadir / "test_audio_44100.wav") + data = {"audio": [audio_path]} +--- a/tests/test_metric_common.py 2023-05-06 13:20:24.496197629 +0200 ++++ b/tests/test_metric_common.py 2023-05-06 13:21:09.916732417 +0200 +@@ -210,6 +210,7 @@ + yield + + ++@pytest.mark.skip(reason="require seqeval") + def test_seqeval_raises_when_incorrect_scheme(): + metric = load_metric(os.path.join("metrics", "seqeval")) + wrong_scheme = "ERROR" +--- a/tests/packaged_modules/test_audiofolder.py 2023-05-06 14:00:39.560876163 +0200 ++++ b/tests/packaged_modules/test_audiofolder.py 2023-05-06 14:01:26.005212423 +0200 +@@ -4,7 +4,6 @@ + import librosa + import numpy as np + import pytest +-import soundfile as sf + + from datasets import Audio, ClassLabel, Features, Value + from datasets.data_files import DataFilesDict, get_data_patterns_locally +@@ -191,9 +190,11 @@ + assert len(data_files_with_two_splits_and_metadata["test"]) == 2 + return data_files_with_two_splits_and_metadata + +- ++@pytest.mark.skip(reason="require soundfile") + @pytest.fixture + def data_files_with_zip_archives(tmp_path, audio_file): ++ import soundfile as sf ++ + data_dir = tmp_path / "audiofolder_data_dir_with_zip_archives" + data_dir.mkdir(parents=True, exist_ok=True) + archive_dir = data_dir / "archive" +--- a/tests/test_arrow_dataset.py 2023-05-06 15:36:11.080459079 +0200 ++++ b/tests/test_arrow_dataset.py 2023-05-06 15:38:07.452828528 +0200 +@@ -3928,6 +3928,7 @@ + ) + self.assertDictEqual(features_after_cast, dset.features) + ++ @pytest.mark.skip(reason="require soundfile") + def test_task_automatic_speech_recognition(self): + # Include a dummy extra column `dummy` to test we drop it correctly + features_before_cast = Features( diff --git a/sci-libs/datasets/metadata.xml b/sci-libs/datasets/metadata.xml new file mode 100644 index 000000000000..94c112402049 --- /dev/null +++ b/sci-libs/datasets/metadata.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd"> +<pkgmetadata> + <maintainer type="person"> + <email>tup...@gentoo.org</email> + <name>Tupone Alfredo</name> + </maintainer> + <upstream> + <remote-id type="github">huggingface/datasets</remote-id> + <remote-id type="pypi">datasets</remote-id> + </upstream> +</pkgmetadata>