commit: 6d7268c6c8e47b43efd5550b05d3ad587704bdff
Author: Alfredo Tupone <tupone <AT> gentoo <DOT> org>
AuthorDate: Sun May 7 20:15:13 2023 +0000
Commit: Alfredo Tupone <tupone <AT> gentoo <DOT> org>
CommitDate: Sun May 7 20:15:42 2023 +0000
URL: https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=6d7268c6
sci-libs/datasets: new package, add 2.11.0
Signed-off-by: Alfredo Tupone <tupone <AT> gentoo.org>
sci-libs/datasets/Manifest | 1 +
sci-libs/datasets/datasets-2.11.0.ebuild | 53 +++++++
.../datasets/files/datasets-2.11.0-tests.patch | 168 +++++++++++++++++++++
sci-libs/datasets/metadata.xml | 12 ++
4 files changed, 234 insertions(+)
diff --git a/sci-libs/datasets/Manifest b/sci-libs/datasets/Manifest
new file mode 100644
index 000000000000..37284ce3b3e7
--- /dev/null
+++ b/sci-libs/datasets/Manifest
@@ -0,0 +1 @@
+DIST datasets-2.11.0.gh.tar.gz 2141289 BLAKE2B
0fb471dd6ee5de3831eb6586c4a15e67381262470b72d5ab02ee87dfc7977cb4d40e04da6507049d1e47cb8948cad11988bb7627293b48231e1cd413d2cfb885
SHA512
9ec2274d7978e3dde1b2f8ce78dd65bdf66742bbfee7b8672af46216aeaae3ef5c4604a8a5ea0bdee808f1c362cca9a122c16d2e9a161678148e581e4cd5c863
diff --git a/sci-libs/datasets/datasets-2.11.0.ebuild
b/sci-libs/datasets/datasets-2.11.0.ebuild
new file mode 100644
index 000000000000..0ff1a4f0b386
--- /dev/null
+++ b/sci-libs/datasets/datasets-2.11.0.ebuild
@@ -0,0 +1,53 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+DISTUTILS_USE_PEP517=setuptools
+PYTHON_COMPAT=( python3_{9..11} )
+DISTUTILS_SINGLE_IMPL=1
+inherit distutils-r1
+
+DESCRIPTION="access and share datasets for Audio, Computer Vision, and NLP
tasks"
+HOMEPAGE="
+ https://pypi.org/project/datasets/
+"
+SRC_URI="https://github.com/huggingface/${PN}/archive/refs/tags/${PV}.tar.gz
+ -> ${P}.gh.tar.gz"
+IUSE="test"
+
+LICENSE="Apache-2.0"
+SLOT="0"
+KEYWORDS="~amd64"
+
+RDEPEND="
+ ${PYTHON_DEPS}
+ sci-libs/pytorch[${PYTHON_SINGLE_USEDEP}]
+ $(python_gen_cond_dep '
+ dev-python/pyarrow[${PYTHON_USEDEP},parquet,snappy]
+ dev-python/fsspec[${PYTHON_USEDEP}]
+ dev-python/multiprocess[${PYTHON_USEDEP}]
+ dev-python/aiohttp[${PYTHON_USEDEP}]
+ dev-python/xxhash[${PYTHON_USEDEP}]
+ dev-python/zstandard[${PYTHON_USEDEP}]
+ dev-python/absl-py[${PYTHON_USEDEP}]
+ sci-libs/scikit-learn[${PYTHON_USEDEP}]
+ ')
+"
+DEPEND="${RDEPEND}"
+BDEPEND="test? (
+ $(python_gen_cond_dep '
+ dev-python/pytest-datadir[${PYTHON_USEDEP}]
+ ')
+)"
+
+PATCHES=( "${FILESDIR}"/${P}-tests.patch )
+
+distutils_enable_tests pytest
+
+src_prepare() {
+ # Require jiwer
+ rm metrics/cer/test_cer.py || die
+
+ distutils-r1_src_prepare
+}
diff --git a/sci-libs/datasets/files/datasets-2.11.0-tests.patch
b/sci-libs/datasets/files/datasets-2.11.0-tests.patch
new file mode 100644
index 000000000000..01e5d9c70e7b
--- /dev/null
+++ b/sci-libs/datasets/files/datasets-2.11.0-tests.patch
@@ -0,0 +1,168 @@
+--- a/tests/test_metric_common.py 2023-05-04 18:48:48.550861318 +0200
++++ b/tests/test_metric_common.py 2023-05-04 18:50:25.787364577 +0200
+@@ -93,6 +93,7 @@
+ INTENSIVE_CALLS_PATCHER = {}
+ metric_name = None
+
++ @pytest.mark.skip(reason="disabling, depends on bert_score, bleurt,
math_equivalence, coval, nltk, faiss, mauve, rouge_score, sacrebleu, sacremoses
...")
+ def test_load_metric(self, metric_name):
+ doctest.ELLIPSIS_MARKER = "[...]"
+ metric_module = importlib.import_module(
+--- a/tests/test_hf_gcp.py 2023-05-04 19:33:31.150825303 +0200
++++ b/tests/test_hf_gcp.py 2023-05-04 19:40:08.401759538 +0200
+@@ -69,6 +69,7 @@
+ self.assertTrue(os.path.exists(datset_info_path))
+
+
++@pytest.mark.skip(reason="require apache_beam")
+ @pytest.mark.integration
+ def test_wikipedia_frr(tmp_path_factory):
+ tmp_dir = tmp_path_factory.mktemp("test_hf_gcp") / "test_wikipedia_simple"
+--- a/tests/test_distributed.py 2023-05-04 19:43:09.861275030 +0200
++++ b/tests/test_distributed.py 2023-05-04 19:44:17.608326722 +0200
+@@ -55,6 +55,7 @@
+ assert len({tuple(x.values()) for ds in datasets_per_rank for x in ds})
== full_size
+
+
++@pytest.mark.skip(reason="require distributed torch")
+ @pytest.mark.parametrize("streaming", [False, True])
+ @require_torch
+ @pytest.mark.skipif(os.name == "nt", reason="execute_subprocess_async doesn't
support windows")
+@@ -76,6 +77,7 @@
+ execute_subprocess_async(cmd, env=os.environ.copy())
+
+
++@pytest.mark.skip(reason="require distributed torch")
+ @pytest.mark.parametrize(
+ "nproc_per_node, num_workers",
+ [
+--- a/tests/utils.py 2023-05-06 08:43:16.251987543 +0200
++++ b/tests/utils.py 2023-05-06 08:44:24.467952870 +0200
+@@ -54,8 +54,8 @@
+ # Audio
+ require_sndfile = pytest.mark.skipif(
+ # On Windows and OS X, soundfile installs sndfile
+- find_spec("soundfile") is None or
version.parse(importlib_metadata.version("soundfile")) <
version.parse("0.12.0"),
+- reason="test requires sndfile>=0.12.1: 'pip install
\"soundfile>=0.12.1\"'; ",
++ True,
++ reason="test requires librosa",
+ )
+
+ # Beam
+--- a/tests/features/test_audio.py 2023-05-06 09:03:58.680108142 +0200
++++ a/tests/features/test_audio.py 2023-05-06 09:05:50.463407967 +0200
+@@ -57,6 +57,7 @@
+ assert features.arrow_schema == pa.schema({"sequence_of_audios":
pa.list_(Audio().pa_type)})
+
+
++@pytest.mark.skip(reason="require librosa")
+ @pytest.mark.parametrize(
+ "build_example",
+ [
+@@ -82,6 +82,7 @@
+ assert decoded_example.keys() == {"path", "array", "sampling_rate"}
+
+
++@pytest.mark.skip(reason="require librosa")
+ @pytest.mark.parametrize(
+ "build_example",
+ [
+@@ -148,6 +149,7 @@
+ assert decoded_example["sampling_rate"] == 48000
+
+
++@pytest.mark.skip(reason="require librosa")
+ @pytest.mark.parametrize("sampling_rate", [16_000, 48_000])
+ def test_audio_decode_example_pcm(shared_datadir, sampling_rate):
+ audio_path = str(shared_datadir / "test_audio_16000.pcm")
+@@ -416,6 +417,7 @@
+ assert column[0]["sampling_rate"] == 16000
+
+
++@pytest.mark.skip(reason="require librosa")
+ @pytest.mark.parametrize(
+ "build_data",
+ [
+@@ -440,6 +442,7 @@
+ assert item["audio"].keys() == {"path", "array", "sampling_rate"}
+
+
++@pytest.mark.skip(reason="require librosa")
+ def test_dataset_concatenate_audio_features(shared_datadir):
+ # we use a different data structure between 1 and 2 to make sure they are
compatible with each other
+ audio_path = str(shared_datadir / "test_audio_44100.wav")
+@@ -453,6 +456,7 @@
+ assert concatenated_dataset[1]["audio"]["array"].shape ==
dset2[0]["audio"]["array"].shape
+
+
++@pytest.mark.skip(reason="require librosa")
+ def test_dataset_concatenate_nested_audio_features(shared_datadir):
+ # we use a different data structure between 1 and 2 to make sure they are
compatible with each other
+ audio_path = str(shared_datadir / "test_audio_44100.wav")
+@@ -616,6 +616,7 @@
+ assert isinstance(ds, Dataset)
+
+
++@require_sndfile
+ def test_dataset_with_audio_feature_undecoded(shared_datadir):
+ audio_path = str(shared_datadir / "test_audio_44100.wav")
+ data = {"audio": [audio_path]}
+@@ -633,6 +634,7 @@
+ assert column[0] == {"path": audio_path, "bytes": None}
+
+
++@require_sndfile
+ def test_formatted_dataset_with_audio_feature_undecoded(shared_datadir):
+ audio_path = str(shared_datadir / "test_audio_44100.wav")
+ data = {"audio": [audio_path]}
+@@ -664,6 +666,7 @@
+ assert column[0] == {"path": audio_path, "bytes": None}
+
+
++@require_sndfile
+ def test_dataset_with_audio_feature_map_undecoded(shared_datadir):
+ audio_path = str(shared_datadir / "test_audio_44100.wav")
+ data = {"audio": [audio_path]}
+--- a/tests/test_metric_common.py 2023-05-06 13:20:24.496197629 +0200
++++ b/tests/test_metric_common.py 2023-05-06 13:21:09.916732417 +0200
+@@ -210,6 +210,7 @@
+ yield
+
+
++@pytest.mark.skip(reason="require seqeval")
+ def test_seqeval_raises_when_incorrect_scheme():
+ metric = load_metric(os.path.join("metrics", "seqeval"))
+ wrong_scheme = "ERROR"
+--- a/tests/packaged_modules/test_audiofolder.py 2023-05-06
14:00:39.560876163 +0200
++++ b/tests/packaged_modules/test_audiofolder.py 2023-05-06
14:01:26.005212423 +0200
+@@ -4,7 +4,6 @@
+ import librosa
+ import numpy as np
+ import pytest
+-import soundfile as sf
+
+ from datasets import Audio, ClassLabel, Features, Value
+ from datasets.data_files import DataFilesDict, get_data_patterns_locally
+@@ -191,9 +190,11 @@
+ assert len(data_files_with_two_splits_and_metadata["test"]) == 2
+ return data_files_with_two_splits_and_metadata
+
+-
++@pytest.mark.skip(reason="require soundfile")
+ @pytest.fixture
+ def data_files_with_zip_archives(tmp_path, audio_file):
++ import soundfile as sf
++
+ data_dir = tmp_path / "audiofolder_data_dir_with_zip_archives"
+ data_dir.mkdir(parents=True, exist_ok=True)
+ archive_dir = data_dir / "archive"
+--- a/tests/test_arrow_dataset.py 2023-05-06 15:36:11.080459079 +0200
++++ b/tests/test_arrow_dataset.py 2023-05-06 15:38:07.452828528 +0200
+@@ -3928,6 +3928,7 @@
+ )
+ self.assertDictEqual(features_after_cast, dset.features)
+
++ @pytest.mark.skip(reason="require soundfile")
+ def test_task_automatic_speech_recognition(self):
+ # Include a dummy extra column `dummy` to test we drop it correctly
+ features_before_cast = Features(
diff --git a/sci-libs/datasets/metadata.xml b/sci-libs/datasets/metadata.xml
new file mode 100644
index 000000000000..94c112402049
--- /dev/null
+++ b/sci-libs/datasets/metadata.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd";>
+<pkgmetadata>
+ <maintainer type="person">
+ <email>tup...@gentoo.org</email>
+ <name>Tupone Alfredo</name>
+ </maintainer>
+ <upstream>
+ <remote-id type="github">huggingface/datasets</remote-id>
+ <remote-id type="pypi">datasets</remote-id>
+ </upstream>
+</pkgmetadata>
