https://github.com/python/cpython/commit/08cacb2a14dd2db133448b24bb9c1d7d21bed07d
commit: 08cacb2a14dd2db133448b24bb9c1d7d21bed07d
branch: 3.13
author: Mikhail Efimov <[email protected]>
committer: pablogsal <[email protected]>
date: 2025-10-29T14:33:55Z
summary:
[3.13] gh-140576: Fixed crash produced by lexer in case of dedented zero byte
(GH-140583) (#140762)
files:
A Misc/NEWS.d/next/Core and
Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
M Lib/test/test_tokenize.py
M Parser/lexer/lexer.c
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 9aff15eb530845..4a7b02ee05b895 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -3097,6 +3097,7 @@ def get_tokens(string):
f'__{
x:d
}__'""",
+ " a\n\x00",
]:
with self.subTest(case=case):
self.assertRaises(tokenize.TokenError, get_tokens, case)
diff --git a/Misc/NEWS.d/next/Core and
Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core
and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
new file mode 100644
index 00000000000000..2c27525d9f782c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and
Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
@@ -0,0 +1,2 @@
+Fixed crash in :func:`tokenize.generate_tokens` in case of
+specific incorrect input. Patch by Mikhail Efimov.
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index 66a7cbb769aa46..cc89f0b9cc9af0 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -481,6 +481,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode*
current_tok, struct t
return MAKE_TOKEN(ERRORTOKEN);
}
}
+ else if (c == EOF && PyErr_Occurred()) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
else {
break;
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
