https://github.com/python/cpython/commit/c45ab05a3ad779ba7316714eacae75d25944cd73 commit: c45ab05a3ad779ba7316714eacae75d25944cd73 branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: pablogsal <[email protected]> date: 2025-10-29T13:54:37Z summary:
[3.14] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH-140583) (#140757) gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH-140583) (cherry picked from commit 8706167474e9a625e5f6613d3c7ac77a62faff58) Co-authored-by: Mikhail Efimov <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst M Lib/test/test_tokenize.py M Parser/lexer/lexer.c diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index d274726eed2e65..ca67e381958757 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3183,6 +3183,7 @@ def get_tokens(string): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst new file mode 100644 index 00000000000000..2c27525d9f782c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst @@ -0,0 +1,2 @@ +Fixed crash in :func:`tokenize.generate_tokens` in case of +specific incorrect input. Patch by Mikhail Efimov. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a69994e9b3d005..7f25afec302c22 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -539,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
