This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 44291877b93 [AINode] Revert transformer and tokenizer dependencies
update (#16394)
44291877b93 is described below
commit 44291877b93fc4f19497e34718eea6c4e539061a
Author: Yongzao <[email protected]>
AuthorDate: Thu Sep 11 19:25:58 2025 +0800
[AINode] Revert transformer and tokenizer dependencies update (#16394)
---
.../iotdb/ainode/it/AINodeInferenceSQLIT.java | 114 ++++++++----
.../iotdb/ainode/core/manager/inference_manager.py | 15 +-
iotdb-core/ainode/poetry.lock | 193 +++++++++++++++------
iotdb-core/ainode/pyproject.toml | 4 +-
4 files changed, 230 insertions(+), 96 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
index c18e24a793a..2fcc180a3e3 100644
---
a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
@@ -56,26 +56,12 @@ public class AINodeInferenceSQLIT {
"CREATE TIMESERIES root.AI.s1 WITH DATATYPE=DOUBLE, ENCODING=RLE",
"CREATE TIMESERIES root.AI.s2 WITH DATATYPE=INT32, ENCODING=RLE",
"CREATE TIMESERIES root.AI.s3 WITH DATATYPE=INT64, ENCODING=RLE",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(1,1.0,2.0,3,4)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(2,2.0,3.0,4,5)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(3,3.0,4.0,5,6)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(4,4.0,5.0,6,7)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(5,5.0,6.0,7,8)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(6,6.0,7.0,8,9)",
- "insert into root.AI(timestamp,s0,s1,s2,s3) values(7,7.0,8.0,9,10)",
};
static String[] WRITE_SQL_IN_TABLE =
new String[] {
"CREATE DATABASE root",
"CREATE TABLE root.AI (s0 FLOAT FIELD, s1 DOUBLE FIELD, s2 INT32
FIELD, s3 INT64 FIELD)",
- "insert into root.AI(time,s0,s1,s2,s3) values(1,1.0,2.0,3,4)",
- "insert into root.AI(time,s0,s1,s2,s3) values(2,2.0,3.0,4,5)",
- "insert into root.AI(time,s0,s1,s2,s3) values(3,3.0,4.0,5,6)",
- "insert into root.AI(time,s0,s1,s2,s3) values(4,4.0,5.0,6,7)",
- "insert into root.AI(time,s0,s1,s2,s3) values(5,5.0,6.0,7,8)",
- "insert into root.AI(time,s0,s1,s2,s3) values(6,6.0,7.0,8,9)",
- "insert into root.AI(time,s0,s1,s2,s3) values(7,7.0,8.0,9,10)",
};
@BeforeClass
@@ -84,6 +70,24 @@ public class AINodeInferenceSQLIT {
EnvFactory.getEnv().initClusterEnvironment(1, 1);
prepareData(WRITE_SQL_IN_TREE);
prepareTableData(WRITE_SQL_IN_TABLE);
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ for (int i = 0; i < 2880; i++) {
+ statement.execute(
+ String.format(
+ "INSERT INTO root.AI(timestamp,s0,s1,s2,s3)
VALUES(%d,%f,%f,%d,%d)",
+ i, (float) i, (double) i, i, i));
+ }
+ }
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ for (int i = 0; i < 2880; i++) {
+ statement.execute(
+ String.format(
+ "INSERT INTO root.AI(time,s0,s1,s2,s3) VALUES(%d,%f,%f,%d,%d)",
+ i, (float) i, (double) i, i, i));
+ }
+ }
}
@AfterClass
@@ -109,6 +113,29 @@ public class AINodeInferenceSQLIT {
}
public void callInferenceTest(Statement statement) throws SQLException {
+ // SQL0: Invoke timer-sundial and timer-xl to inference, the result should
success
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "CALL INFERENCE(sundial, \"select s1 from root.AI\",
generateTime=true, predict_length=720)")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ checkHeader(resultSetMetaData, "Time,output0");
+ int count = 0;
+ while (resultSet.next()) {
+ count++;
+ }
+ assertEquals(720, count);
+ }
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "CALL INFERENCE(timer_xl, \"select s2 from root.AI\",
generateTime=true, predict_length=256)")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ checkHeader(resultSetMetaData, "Time,output0");
+ int count = 0;
+ while (resultSet.next()) {
+ count++;
+ }
+ assertEquals(256, count);
+ }
// SQL1: user-defined model inferences multi-columns with generateTime=true
String sql1 =
"CALL INFERENCE(identity, \"select s0,s1,s2,s3 from root.AI\",
generateTime=true)";
@@ -171,15 +198,15 @@ public class AINodeInferenceSQLIT {
// assertEquals(3, count);
// }
- try (ResultSet resultSet = statement.executeQuery(sql4)) {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- checkHeader(resultSetMetaData, "Time,output0");
- int count = 0;
- while (resultSet.next()) {
- count++;
- }
- assertEquals(6, count);
- }
+ // try (ResultSet resultSet = statement.executeQuery(sql4)) {
+ // ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ // checkHeader(resultSetMetaData, "Time,output0");
+ // int count = 0;
+ // while (resultSet.next()) {
+ // count++;
+ // }
+ // assertEquals(6, count);
+ // }
}
@Test
@@ -219,6 +246,29 @@ public class AINodeInferenceSQLIT {
public void selectForecastTestInTable() throws SQLException {
try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
Statement statement = connection.createStatement()) {
+ // SQL0: Invoke timer-sundial and timer-xl to forecast, the result
should success
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT * FROM FORECAST(model_id=>'sundial', input=>(SELECT
time,s1 FROM root.AI) ORDER BY time, output_length=>720)")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ checkHeader(resultSetMetaData, "time,s1");
+ int count = 0;
+ while (resultSet.next()) {
+ count++;
+ }
+ assertEquals(720, count);
+ }
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT * FROM FORECAST(model_id=>'timer_xl', input=>(SELECT
time,s2 FROM root.AI) ORDER BY time, output_length=>256)")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ checkHeader(resultSetMetaData, "time,s2");
+ int count = 0;
+ while (resultSet.next()) {
+ count++;
+ }
+ assertEquals(256, count);
+ }
// SQL1: user-defined model inferences multi-columns with
generateTime=true
String sql1 =
"SELECT * FROM FORECAST(model_id=>'identity', input=>(SELECT
time,s0,s1,s2,s3 FROM root.AI) ORDER BY time, output_length=>7)";
@@ -280,15 +330,15 @@ public class AINodeInferenceSQLIT {
// assertEquals(3, count);
// }
- try (ResultSet resultSet = statement.executeQuery(sql4)) {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- checkHeader(resultSetMetaData, "time,s0");
- int count = 0;
- while (resultSet.next()) {
- count++;
- }
- assertEquals(6, count);
- }
+ // try (ResultSet resultSet = statement.executeQuery(sql4)) {
+ // ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ // checkHeader(resultSetMetaData, "time,s0");
+ // int count = 0;
+ // while (resultSet.next()) {
+ // count++;
+ // }
+ // assertEquals(6, count);
+ // }
}
}
}
diff --git a/iotdb-core/ainode/iotdb/ainode/core/manager/inference_manager.py
b/iotdb-core/ainode/iotdb/ainode/core/manager/inference_manager.py
index db4995c55ba..81c414387a3 100644
--- a/iotdb-core/ainode/iotdb/ainode/core/manager/inference_manager.py
+++ b/iotdb-core/ainode/iotdb/ainode/core/manager/inference_manager.py
@@ -82,7 +82,8 @@ class TimerXLStrategy(InferenceStrategy):
def infer(self, full_data, predict_length=96, **_):
data = full_data[1][0]
if data.dtype.byteorder not in ("=", "|"):
- data = data.byteswap().newbyteorder()
+ np_data = data.byteswap()
+ data = np_data.view(np_data.dtype.newbyteorder())
seqs = torch.tensor(data).unsqueeze(0).float()
# TODO: unify model inference input
output = self.model.generate(seqs, max_new_tokens=predict_length,
revin=True)
@@ -94,7 +95,8 @@ class SundialStrategy(InferenceStrategy):
def infer(self, full_data, predict_length=96, **_):
data = full_data[1][0]
if data.dtype.byteorder not in ("=", "|"):
- data = data.byteswap().newbyteorder()
+ np_data = data.byteswap()
+ data = np_data.view(np_data.dtype.newbyteorder())
seqs = torch.tensor(data).unsqueeze(0).float()
# TODO: unify model inference input
output = self.model.generate(
@@ -228,7 +230,7 @@ class InferenceManager:
full_data = deserializer(raw)
inference_attrs = extract_attrs(req)
- predict_length = int(inference_attrs.get("predict_length", 96))
+ predict_length = int(inference_attrs.pop("predict_length", 96))
if (
predict_length
>
AINodeDescriptor().get_config().get_ain_inference_max_predict_length()
@@ -249,7 +251,8 @@ class InferenceManager:
# TODO: TSBlock -> Tensor codes should be unified
data = full_data[1][0]
if data.dtype.byteorder not in ("=", "|"):
- data = data.byteswap().newbyteorder()
+ np_data = data.byteswap()
+ data = np_data.view(np_data.dtype.newbyteorder())
# the inputs should be on CPU before passing to the inference
request
inputs = torch.tensor(data).unsqueeze(0).float().to("cpu")
if model_id == "sundial":
@@ -275,7 +278,9 @@ class InferenceManager:
model = self._model_manager.load_model(model_id,
inference_attrs, accel)
# inference by strategy
strategy = self._get_strategy(model_id, model)
- outputs = strategy.infer(full_data, **inference_attrs)
+ outputs = strategy.infer(
+ full_data, predict_length=predict_length, **inference_attrs
+ )
# construct response
status = get_status(TSStatusCode.SUCCESS_STATUS)
diff --git a/iotdb-core/ainode/poetry.lock b/iotdb-core/ainode/poetry.lock
index e4b02e12c04..ee4750470ff 100644
--- a/iotdb-core/ainode/poetry.lock
+++ b/iotdb-core/ainode/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.4 and should not be
changed by hand.
+# This file is automatically @generated by Poetry 2.1.2 and should not be
changed by hand.
[[package]]
name = "alembic"
@@ -1872,27 +1872,112 @@ twisted = ["twisted"]
[[package]]
name = "tokenizers"
-version = "0.22.0"
+version = "0.19.1"
description = ""
optional = false
-python-versions = ">=3.9"
+python-versions = ">=3.7"
groups = ["main"]
files = [
- {file = "tokenizers-0.22.0-cp39-abi3-macosx_10_12_x86_64.whl", hash =
"sha256:eaa9620122a3fb99b943f864af95ed14c8dfc0f47afa3b404ac8c16b3f2bb484"},
- {file = "tokenizers-0.22.0-cp39-abi3-macosx_11_0_arm64.whl", hash =
"sha256:71784b9ab5bf0ff3075bceeb198149d2c5e068549c0d18fe32d06ba0deb63f79"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:ec5b71f668a8076802b0241a42387d48289f25435b86b769ae1837cad4172a17"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:ea8562fa7498850d02a16178105b58803ea825b50dc9094d60549a7ed63654bb"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash
= "sha256:4136e1558a9ef2e2f1de1555dcd573e1cbc4a320c1a06c4107a3d46dc8ac6e4b"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:cdf5954de3962a5fd9781dc12048d24a1a6f1f5df038c6e95db328cd22964206"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:8337ca75d0731fc4860e6204cc24bb36a67d9736142aa06ed320943b50b1e7ed"},
- {file =
"tokenizers-0.22.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:a89264e26f63c449d8cded9061adea7b5de53ba2346fc7e87311f7e4117c1cc8"},
- {file = "tokenizers-0.22.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash =
"sha256:790bad50a1b59d4c21592f9c3cf5e5cf9c3c7ce7e1a23a739f13e01fb1be377a"},
- {file = "tokenizers-0.22.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash =
"sha256:76cf6757c73a10ef10bf06fa937c0ec7393d90432f543f49adc8cab3fb6f26cb"},
- {file = "tokenizers-0.22.0-cp39-abi3-musllinux_1_2_i686.whl", hash =
"sha256:1626cb186e143720c62c6c6b5371e62bbc10af60481388c0da89bc903f37ea0c"},
- {file = "tokenizers-0.22.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash =
"sha256:da589a61cbfea18ae267723d6b029b84598dc8ca78db9951d8f5beff72d8507c"},
- {file = "tokenizers-0.22.0-cp39-abi3-win32.whl", hash =
"sha256:dbf9d6851bddae3e046fedfb166f47743c1c7bd11c640f0691dd35ef0bcad3be"},
- {file = "tokenizers-0.22.0-cp39-abi3-win_amd64.whl", hash =
"sha256:c78174859eeaee96021f248a56c801e36bfb6bd5b067f2e95aa82445ca324f00"},
- {file = "tokenizers-0.22.0.tar.gz", hash =
"sha256:2e33b98525be8453f355927f3cab312c36cd3e44f4d7e9e97da2fa94d0a49dcb"},
+ {file = "tokenizers-0.19.1-cp310-cp310-macosx_10_12_x86_64.whl", hash =
"sha256:952078130b3d101e05ecfc7fc3640282d74ed26bcf691400f872563fca15ac97"},
+ {file = "tokenizers-0.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash =
"sha256:82c8b8063de6c0468f08e82c4e198763e7b97aabfe573fd4cf7b33930ca4df77"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:f03727225feaf340ceeb7e00604825addef622d551cbd46b7b775ac834c1e1c4"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:453e4422efdfc9c6b6bf2eae00d5e323f263fff62b29a8c9cd526c5003f3f642"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:02e81bf089ebf0e7f4df34fa0207519f07e66d8491d963618252f2e0729e0b46"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:b07c538ba956843833fee1190cf769c60dc62e1cf934ed50d77d5502194d63b1"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:e28cab1582e0eec38b1f38c1c1fb2e56bce5dc180acb1724574fc5f47da2a4fe"},
+ {file =
"tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:8b01afb7193d47439f091cd8f070a1ced347ad0f9144952a30a41836902fe09e"},
+ {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash =
"sha256:7fb297edec6c6841ab2e4e8f357209519188e4a59b557ea4fafcf4691d1b4c98"},
+ {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash =
"sha256:2e8a3dd055e515df7054378dc9d6fa8c8c34e1f32777fb9a01fea81496b3f9d3"},
+ {file = "tokenizers-0.19.1-cp310-none-win32.whl", hash =
"sha256:7ff898780a155ea053f5d934925f3902be2ed1f4d916461e1a93019cc7250837"},
+ {file = "tokenizers-0.19.1-cp310-none-win_amd64.whl", hash =
"sha256:bea6f9947e9419c2fda21ae6c32871e3d398cba549b93f4a65a2d369662d9403"},
+ {file = "tokenizers-0.19.1-cp311-cp311-macosx_10_12_x86_64.whl", hash =
"sha256:5c88d1481f1882c2e53e6bb06491e474e420d9ac7bdff172610c4f9ad3898059"},
+ {file = "tokenizers-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash =
"sha256:ddf672ed719b4ed82b51499100f5417d7d9f6fb05a65e232249268f35de5ed14"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:dadc509cc8a9fe460bd274c0e16ac4184d0958117cf026e0ea8b32b438171594"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:dfedf31824ca4915b511b03441784ff640378191918264268e6923da48104acc"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:ac11016d0a04aa6487b1513a3a36e7bee7eec0e5d30057c9c0408067345c48d2"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:76951121890fea8330d3a0df9a954b3f2a37e3ec20e5b0530e9a0044ca2e11fe"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:b342d2ce8fc8d00f376af068e3274e2e8649562e3bc6ae4a67784ded6b99428d"},
+ {file =
"tokenizers-0.19.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:d16ff18907f4909dca9b076b9c2d899114dd6abceeb074eca0c93e2353f943aa"},
+ {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash =
"sha256:706a37cc5332f85f26efbe2bdc9ef8a9b372b77e4645331a405073e4b3a8c1c6"},
+ {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash =
"sha256:16baac68651701364b0289979ecec728546133e8e8fe38f66fe48ad07996b88b"},
+ {file = "tokenizers-0.19.1-cp311-none-win32.whl", hash =
"sha256:9ed240c56b4403e22b9584ee37d87b8bfa14865134e3e1c3fb4b2c42fafd3256"},
+ {file = "tokenizers-0.19.1-cp311-none-win_amd64.whl", hash =
"sha256:ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66"},
+ {file = "tokenizers-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl", hash =
"sha256:621d670e1b1c281a1c9698ed89451395d318802ff88d1fc1accff0867a06f153"},
+ {file = "tokenizers-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash =
"sha256:d924204a3dbe50b75630bd16f821ebda6a5f729928df30f582fb5aade90c818a"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:4f3fefdc0446b1a1e6d81cd4c07088ac015665d2e812f6dbba4a06267d1a2c95"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:9620b78e0b2d52ef07b0d428323fb34e8ea1219c5eac98c2596311f20f1f9266"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:04ce49e82d100594715ac1b2ce87d1a36e61891a91de774755f743babcd0dd52"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:c5c2ff13d157afe413bf7e25789879dd463e5a4abfb529a2d8f8473d8042e28f"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:3174c76efd9d08f836bfccaca7cfec3f4d1c0a4cf3acbc7236ad577cc423c840"},
+ {file =
"tokenizers-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:7c9d5b6c0e7a1e979bec10ff960fae925e947aab95619a6fdb4c1d8ff3708ce3"},
+ {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash =
"sha256:a179856d1caee06577220ebcfa332af046d576fb73454b8f4d4b0ba8324423ea"},
+ {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash =
"sha256:952b80dac1a6492170f8c2429bd11fcaa14377e097d12a1dbe0ef2fb2241e16c"},
+ {file = "tokenizers-0.19.1-cp312-none-win32.whl", hash =
"sha256:01d62812454c188306755c94755465505836fd616f75067abcae529c35edeb57"},
+ {file = "tokenizers-0.19.1-cp312-none-win_amd64.whl", hash =
"sha256:b70bfbe3a82d3e3fb2a5e9b22a39f8d1740c96c68b6ace0086b39074f08ab89a"},
+ {file = "tokenizers-0.19.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash =
"sha256:bb9dfe7dae85bc6119d705a76dc068c062b8b575abe3595e3c6276480e67e3f1"},
+ {file = "tokenizers-0.19.1-cp37-cp37m-macosx_11_0_arm64.whl", hash =
"sha256:1f0360cbea28ea99944ac089c00de7b2e3e1c58f479fb8613b6d8d511ce98267"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash
= "sha256:71e3ec71f0e78780851fef28c2a9babe20270404c921b756d7c532d280349214"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:b82931fa619dbad979c0ee8e54dd5278acc418209cc897e42fac041f5366d626"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:e8ff5b90eabdcdaa19af697885f70fe0b714ce16709cf43d4952f1f85299e73a"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:e742d76ad84acbdb1a8e4694f915fe59ff6edc381c97d6dfdd054954e3478ad4"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:d8c5d59d7b59885eab559d5bc082b2985555a54cda04dda4c65528d90ad252ad"},
+ {file =
"tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:6b2da5c32ed869bebd990c9420df49813709e953674c0722ff471a116d97b22d"},
+ {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash =
"sha256:638e43936cc8b2cbb9f9d8dde0fe5e7e30766a3318d2342999ae27f68fdc9bd6"},
+ {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash =
"sha256:78e769eb3b2c79687d9cb0f89ef77223e8e279b75c0a968e637ca7043a84463f"},
+ {file = "tokenizers-0.19.1-cp37-none-win32.whl", hash =
"sha256:72791f9bb1ca78e3ae525d4782e85272c63faaef9940d92142aa3eb79f3407a3"},
+ {file = "tokenizers-0.19.1-cp37-none-win_amd64.whl", hash =
"sha256:f3bbb7a0c5fcb692950b041ae11067ac54826204318922da754f908d95619fbc"},
+ {file = "tokenizers-0.19.1-cp38-cp38-macosx_10_12_x86_64.whl", hash =
"sha256:07f9295349bbbcedae8cefdbcfa7f686aa420be8aca5d4f7d1ae6016c128c0c5"},
+ {file = "tokenizers-0.19.1-cp38-cp38-macosx_11_0_arm64.whl", hash =
"sha256:10a707cc6c4b6b183ec5dbfc5c34f3064e18cf62b4a938cb41699e33a99e03c1"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash
= "sha256:6309271f57b397aa0aff0cbbe632ca9d70430839ca3178bf0f06f825924eca22"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:4ad23d37d68cf00d54af184586d79b84075ada495e7c5c0f601f051b162112dc"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:427c4f0f3df9109314d4f75b8d1f65d9477033e67ffaec4bca53293d3aca286d"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:e83a31c9cf181a0a3ef0abad2b5f6b43399faf5da7e696196ddd110d332519ee"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:c27b99889bd58b7e301468c0838c5ed75e60c66df0d4db80c08f43462f82e0d3"},
+ {file =
"tokenizers-0.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:bac0b0eb952412b0b196ca7a40e7dce4ed6f6926489313414010f2e6b9ec2adf"},
+ {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash =
"sha256:8a6298bde623725ca31c9035a04bf2ef63208d266acd2bed8c2cb7d2b7d53ce6"},
+ {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash =
"sha256:08a44864e42fa6d7d76d7be4bec62c9982f6f6248b4aa42f7302aa01e0abfd26"},
+ {file = "tokenizers-0.19.1-cp38-none-win32.whl", hash =
"sha256:1de5bc8652252d9357a666e609cb1453d4f8e160eb1fb2830ee369dd658e8975"},
+ {file = "tokenizers-0.19.1-cp38-none-win_amd64.whl", hash =
"sha256:0bcce02bf1ad9882345b34d5bd25ed4949a480cf0e656bbd468f4d8986f7a3f1"},
+ {file = "tokenizers-0.19.1-cp39-cp39-macosx_10_12_x86_64.whl", hash =
"sha256:0b9394bd204842a2a1fd37fe29935353742be4a3460b6ccbaefa93f58a8df43d"},
+ {file = "tokenizers-0.19.1-cp39-cp39-macosx_11_0_arm64.whl", hash =
"sha256:4692ab92f91b87769d950ca14dbb61f8a9ef36a62f94bad6c82cc84a51f76f6a"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash
= "sha256:6258c2ef6f06259f70a682491c78561d492e885adeaf9f64f5389f78aa49a051"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:c85cf76561fbd01e0d9ea2d1cbe711a65400092bc52b5242b16cfd22e51f0c58"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl",
hash =
"sha256:670b802d4d82bbbb832ddb0d41df7015b3e549714c0e77f9bed3e74d42400fbe"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
hash =
"sha256:85aa3ab4b03d5e99fdd31660872249df5e855334b6c333e0bc13032ff4469c4a"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl",
hash =
"sha256:cbf001afbbed111a79ca47d75941e9e5361297a87d186cbfc11ed45e30b5daba"},
+ {file =
"tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:b4c89aa46c269e4e70c4d4f9d6bc644fcc39bb409cb2a81227923404dd6f5227"},
+ {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash =
"sha256:39c1ec76ea1027438fafe16ecb0fb84795e62e9d643444c1090179e63808c69d"},
+ {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash =
"sha256:c2a0d47a89b48d7daa241e004e71fb5a50533718897a4cd6235cb846d511a478"},
+ {file = "tokenizers-0.19.1-cp39-none-win32.whl", hash =
"sha256:61b7fe8886f2e104d4caf9218b157b106207e0f2a4905c9c7ac98890688aabeb"},
+ {file = "tokenizers-0.19.1-cp39-none-win_amd64.whl", hash =
"sha256:f97660f6c43efd3e0bfd3f2e3e5615bf215680bad6ee3d469df6454b8c6e8256"},
+ {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl",
hash =
"sha256:3b11853f17b54c2fe47742c56d8a33bf49ce31caf531e87ac0d7d13d327c9334"},
+ {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash
= "sha256:d26194ef6c13302f446d39972aaa36a1dda6450bc8949f5eb4c27f51191375bd"},
+ {file =
"tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:e8d1ed93beda54bbd6131a2cb363a576eac746d5c26ba5b7556bc6f964425594"},
+ {file =
"tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:ca407133536f19bdec44b3da117ef0d12e43f6d4b56ac4c765f37eca501c7bda"},
+ {file =
"tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:ce05fde79d2bc2e46ac08aacbc142bead21614d937aac950be88dc79f9db9022"},
+ {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl",
hash =
"sha256:35583cd46d16f07c054efd18b5d46af4a2f070a2dd0a47914e66f3ff5efb2b1e"},
+ {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl",
hash =
"sha256:43350270bfc16b06ad3f6f07eab21f089adb835544417afda0f83256a8bf8b75"},
+ {file = "tokenizers-0.19.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash
= "sha256:b4399b59d1af5645bcee2072a463318114c39b8547437a7c2d6a186a1b5a0e2d"},
+ {file =
"tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:6852c5b2a853b8b0ddc5993cd4f33bfffdca4fcc5d52f89dd4b8eada99379285"},
+ {file =
"tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:bcd266ae85c3d39df2f7e7d0e07f6c41a55e9a3123bb11f854412952deacd828"},
+ {file =
"tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:ecb2651956eea2aa0a2d099434134b1b68f1c31f9a5084d6d53f08ed43d45ff2"},
+ {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl",
hash =
"sha256:b279ab506ec4445166ac476fb4d3cc383accde1ea152998509a94d82547c8e2a"},
+ {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl",
hash =
"sha256:89183e55fb86e61d848ff83753f64cded119f5d6e1f553d14ffee3700d0a4a49"},
+ {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash
= "sha256:b2edbc75744235eea94d595a8b70fe279dd42f3296f76d5a86dde1d46e35f574"},
+ {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash =
"sha256:0e64bfde9a723274e9a71630c3e9494ed7b4c0f76a1faacf7fe294cd26f7ae7c"},
+ {file =
"tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:0b5ca92bfa717759c052e345770792d02d1f43b06f9e790ca0a1db62838816f3"},
+ {file =
"tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:6f8a20266e695ec9d7a946a019c1d5ca4eddb6613d4f466888eee04f16eedb85"},
+ {file =
"tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:63c38f45d8f2a2ec0f3a20073cccb335b9f99f73b3c69483cd52ebc75369d8a1"},
+ {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl",
hash =
"sha256:dd26e3afe8a7b61422df3176e06664503d3f5973b94f45d5c45987e1cb711876"},
+ {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl",
hash =
"sha256:eddd5783a4a6309ce23432353cdb36220e25cbb779bfa9122320666508b44b88"},
+ {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash
= "sha256:56ae39d4036b753994476a1b935584071093b55c7a72e3b8288e68c313ca26e7"},
+ {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash =
"sha256:f9939ca7e58c2758c01b40324a59c034ce0cebad18e0d4563a9b1beab3018243"},
+ {file =
"tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl",
hash =
"sha256:6c330c0eb815d212893c67a032e9dc1b38a803eccb32f3e8172c19cc69fbb439"},
+ {file =
"tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
hash =
"sha256:ec11802450a2487cdf0e634b750a04cbdc1c4d066b97d94ce7dd2cb51ebb325b"},
+ {file =
"tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
hash =
"sha256:a2b718f316b596f36e1dae097a7d5b91fc5b85e90bf08b01ff139bd8953b25af"},
+ {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl",
hash =
"sha256:ed69af290c2b65169f0ba9034d1dc39a5db9459b32f1dd8b5f3f32a3fcf06eab"},
+ {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl",
hash =
"sha256:f8a9c828277133af13f3859d1b6bf1c3cb6e9e1637df0e45312e6b7c2e622b1f"},
+ {file = "tokenizers-0.19.1.tar.gz", hash =
"sha256:ee59e6680ed0fdbe6b724cf38bd70400a0c1dd623b07ac729087270caeac88e3"},
]
[package.dependencies]
@@ -1901,7 +1986,7 @@ huggingface-hub = ">=0.16.4,<1.0"
[package.extras]
dev = ["tokenizers[testing]"]
docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"]
-testing = ["black (==22.3)", "datasets", "numpy", "pytest", "pytest-asyncio",
"requests", "ruff"]
+testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"]
[[package]]
name = "torch"
@@ -2019,77 +2104,71 @@ telegram = ["requests"]
[[package]]
name = "transformers"
-version = "4.56.0"
+version = "4.40.1"
description = "State-of-the-art Machine Learning for JAX, PyTorch and
TensorFlow"
optional = false
-python-versions = ">=3.9.0"
+python-versions = ">=3.8.0"
groups = ["main"]
files = [
- {file = "transformers-4.56.0-py3-none-any.whl", hash =
"sha256:bacf539c38dd850690856881c4974321af93a22f2ee96bcc994741a2121d8e71"},
- {file = "transformers-4.56.0.tar.gz", hash =
"sha256:6ca9c3f38aa4da93ebf877db7156368c1c188c7465f09dbe70951e7622e987fa"},
+ {file = "transformers-4.40.1-py3-none-any.whl", hash =
"sha256:9d5ee0c8142a60501faf9e49a0b42f8e9cb8611823bce4f195a9325a6816337e"},
+ {file = "transformers-4.40.1.tar.gz", hash =
"sha256:55e1697e6f18b58273e7117bb469cdffc11be28995462d8d5e422fef38d2de36"},
]
[package.dependencies]
filelock = "*"
-huggingface-hub = ">=0.34.0,<1.0"
+huggingface-hub = ">=0.19.3,<1.0"
numpy = ">=1.17"
packaging = ">=20.0"
pyyaml = ">=5.1"
regex = "!=2019.12.17"
requests = "*"
-safetensors = ">=0.4.3"
-tokenizers = ">=0.22.0,<=0.23.0"
+safetensors = ">=0.4.1"
+tokenizers = ">=0.19,<0.20"
tqdm = ">=4.27"
[package.extras]
-accelerate = ["accelerate (>=0.26.0)"]
-all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av", "codecarbon
(>=2.8.1)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib
(>=0.4.1,<=0.4.13)", "jinja2 (>=3.1.0)", "kenlm", "keras-nlp
(>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)", "librosa",
"mistral-common[opencv] (>=1.6.3)", "num2words", "onnxconverter-common", "optax
(>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode
(>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0. [...]
+accelerate = ["accelerate (>=0.21.0)"]
+agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets
(!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)",
"torch"]
+all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)",
"codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax
(>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp
(>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)",
"optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]
(>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow
(>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", [...]
audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"]
-benchmark = ["optimum-benchmark (>=0.3.0)"]
-chat-template = ["jinja2 (>=3.1.0)"]
-codecarbon = ["codecarbon (>=2.8.1)"]
-deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"]
-deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)",
"beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "deepspeed
(>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "libcst",
"mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "optuna", "parameterized
(>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0)",
"pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich",
"pytest-timeout", "pytest-xdist", "rjieba", "roug [...]
-dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate
(>=0.26.0)", "av", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter
(==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)",
"faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic
(>=1.0.0,<2.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "jinja2
(>=3.1.0)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)",
"libcst", "librosa", "mistral-common[opencv] (>= [...]
-dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)",
"beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "dill
(<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "kenlm", "keras-nlp
(>=0.3.1,<0.14.0)", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)",
"nltk (<=3.8.1)", "onnxconverter-common", "onnxruntime (>=1.4.0)",
"onnxruntime-tools (>=1.4.2)", "pandas (<2.3.0)", "parameterized (>=0.9)",
"phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pyd [...]
-dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate
(>=0.26.0)", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter
(==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)",
"faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "kenlm", "kernels
(>=0.6.1,<=0.9)", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)",
"nltk (<=3.8.1)", "num2words", "onnxruntime (>=1.4.0)", "onnxruntime-tools
(>=1.4.2)", "optuna", "pandas (<2.3.0)", "parame [...]
-flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib
(>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"]
+codecarbon = ["codecarbon (==1.2.0)"]
+deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"]
+deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)",
"beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed
(>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder
(>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil",
"pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist",
"rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)",
"sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sen [...]
+dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate
(>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)",
"cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill
(<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)",
"fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic
(>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib
(>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa" [...]
+dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)",
"beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill
(<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder
(>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa",
"nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools
(>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode
(>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8 [...]
+dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate
(>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter
(==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)",
"faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)",
"ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk",
"onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna",
"parameterized", "phonemizer", "protobuf", "psutil", "pyct [...]
+docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)",
"codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)",
"hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)",
"kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax
(>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode
(>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)",
"sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (< [...]
+docs-specific = ["hf-doc-builder"]
+flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib
(>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"]
flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"]
ftfy = ["ftfy"]
-hf-xet = ["hf-xet"]
-hub-kernels = ["kernels (>=0.6.1,<=0.9)"]
-integrations = ["kernels (>=0.6.1,<=0.9)", "optuna", "ray[tune] (>=2.7.0)",
"sigopt"]
+integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"]
ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)",
"sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)",
"unidic-lite (>=1.0.7)"]
-mistral-common = ["mistral-common[opencv] (>=1.6.3)"]
modelcreation = ["cookiecutter (==1.7.3)"]
natten = ["natten (>=0.14.6,<0.15.0)"]
-num2words = ["num2words"]
onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools
(>=1.4.2)", "tf2onnx"]
onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"]
-open-telemetry = ["opentelemetry-api", "opentelemetry-exporter-otlp",
"opentelemetry-sdk"]
optuna = ["optuna"]
-quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "pandas
(<2.3.0)", "rich", "ruff (==0.11.2)", "urllib3 (<2.0.0)"]
+quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder
(>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"]
ray = ["ray[tune] (>=2.7.0)"]
-retrieval = ["datasets (>=2.15.0)", "faiss-cpu"]
-ruff = ["ruff (==0.11.2)"]
+retrieval = ["datasets (!=2.5.0)", "faiss-cpu"]
sagemaker = ["sagemaker (>=2.31.0)"]
sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"]
-serving = ["accelerate (>=0.26.0)", "fastapi", "openai (>=1.98.0)", "pydantic
(>=2)", "starlette", "torch (>=2.2)", "uvicorn"]
+serving = ["fastapi", "pydantic", "starlette", "uvicorn"]
sigopt = ["sigopt"]
sklearn = ["scikit-learn"]
speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)",
"torchaudio"]
-testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)",
"datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu",
"libcst", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "parameterized
(>=0.9)", "psutil", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio",
"pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout",
"pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff
(==0.11.2)", "sacrebleu (>=1.4.1 [...]
-tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow
(>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"]
-tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)",
"onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability
(<0.24)", "tensorflow-text (<2.16)", "tf2onnx"]
+testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)",
"datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu",
"hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil",
"pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist",
"rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)",
"sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece
(>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"]
+tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow
(>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"]
+tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu
(>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"]
tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"]
-tiktoken = ["blobfile", "tiktoken"]
-timm = ["timm (!=1.0.18,<=1.0.19)"]
-tokenizers = ["tokenizers (>=0.22.0,<=0.23.0)"]
-torch = ["accelerate (>=0.26.0)", "torch (>=2.2)"]
+timm = ["timm"]
+tokenizers = ["tokenizers (>=0.19,<0.20)"]
+torch = ["accelerate (>=0.21.0)", "torch"]
torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)",
"torchaudio"]
torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"]
-torchhub = ["filelock", "huggingface-hub (>=0.34.0,<1.0)",
"importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf",
"regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)",
"tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "tqdm (>=4.27)"]
-video = ["av"]
+torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)",
"importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf",
"regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)",
"tokenizers (>=0.19,<0.20)", "torch", "tqdm (>=4.27)"]
+video = ["av (==9.2.0)", "decord (==0.6.0)"]
vision = ["Pillow (>=10.0.1,<=15.0)"]
[[package]]
@@ -2180,4 +2259,4 @@ zstd = ["zstandard (>=0.18.0)"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<=3.13.5"
-content-hash =
"4086c76240770e1bef605cff4f2229d92c186b1096876dbc9ffacd6c58efa9e8"
+content-hash =
"8d31dadf57a2134038a78bf1918523e0c777c79791abf1b7757897dd80789f54"
diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml
index d08fb35567f..eff9404400d 100644
--- a/iotdb-core/ainode/pyproject.toml
+++ b/iotdb-core/ainode/pyproject.toml
@@ -78,8 +78,8 @@ sktime = "0.38.5"
# ---- DL / HF stack ----
torch = ">=2.7.0"
torchmetrics = "^1.8.0"
-transformers = "==4.56.0"
-tokenizers = "==0.22.0"
+transformers = "==4.40.1"
+tokenizers = ">=0.19,<0.20"
huggingface_hub = "^0.34.4"
safetensors = "^0.6.2"
einops = "^0.8.1"
