Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-nltk for openSUSE:Factory checked in at 2026-08-13 13:16:00 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-nltk (Old) and /work/SRC/openSUSE:Factory/.python-nltk.new.17972 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-nltk" Thu Aug 13 13:16:00 2026 rev:25 rq:1370851 version:3.10.2 Changes: -------- --- /work/SRC/openSUSE:Factory/python-nltk/python-nltk.changes 2026-08-06 16:25:44.439853456 +0200 +++ /work/SRC/openSUSE:Factory/.python-nltk.new.17972/python-nltk.changes 2026-08-13 13:16:58.488419675 +0200 @@ -1,0 +2,10 @@ +Wed Aug 12 12:20:02 UTC 2026 - Daniel Garcia <[email protected]> + +- Update to 3.10.2 (CVE-2026-12372, bsc#1274590): + * Remove inisec.py and document PYTHONSAFEPATH instead + * Skip draft step in release workflow + * Fix symlink escape in FramenetCorpusReader (CWE-59) + * Guard tempfile.gettempdir() when building pathsec allowed roots + * add tests for transitive_closure + +------------------------------------------------------------------- Old: ---- nltk-3.10.1.obscpio New: ---- nltk-3.10.2.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-nltk.spec ++++++ --- /var/tmp/diff_new_pack.u7wiQN/_old 2026-08-13 13:16:59.204452181 +0200 +++ /var/tmp/diff_new_pack.u7wiQN/_new 2026-08-13 13:16:59.208452361 +0200 @@ -19,7 +19,7 @@ %define modname nltk %{?sle15_python_module_pythons} Name: python-nltk -Version: 3.10.1 +Version: 3.10.2 Release: 0 Summary: Natural Language Toolkit License: Apache-2.0 ++++++ _service ++++++ --- /var/tmp/diff_new_pack.u7wiQN/_old 2026-08-13 13:16:59.268455086 +0200 +++ /var/tmp/diff_new_pack.u7wiQN/_new 2026-08-13 13:16:59.276455449 +0200 @@ -3,8 +3,8 @@ <param name="url">https://github.com/nltk/nltk</param> <param name="scm">git</param> <param name="exclude">web/*</param> - <param name="version">3.10.1</param> - <param name="revision">v3.10.1</param> + <param name="version">3.10.2</param> + <param name="revision">v3.10.2</param> </service> <service mode="manual" name="set_version" /> <service mode="buildtime" name="tar" /> ++++++ nltk-3.10.1.obscpio -> nltk-3.10.2.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/.github/workflows/ci.yml new/nltk-3.10.2/.github/workflows/ci.yml --- old/nltk-3.10.1/.github/workflows/ci.yml 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/.github/workflows/ci.yml 2026-08-05 10:16:23.000000000 +0200 @@ -25,7 +25,7 @@ - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.13" # or your chosen version @@ -45,7 +45,7 @@ - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.14" @@ -82,7 +82,7 @@ - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -156,5 +156,7 @@ - name: Run pytest shell: bash + env: + PYTHONSAFEPATH: "1" run: | pytest --numprocesses auto -rsx --doctest-modules nltk diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/.github/workflows/labeler.yml new/nltk-3.10.2/.github/workflows/labeler.yml --- old/nltk-3.10.1/.github/workflows/labeler.yml 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/.github/workflows/labeler.yml 2026-08-05 10:16:23.000000000 +0200 @@ -9,7 +9,7 @@ pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v6 + - uses: actions/labeler@v7 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" sync-labels: true diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/.github/workflows/release.yml new/nltk-3.10.2/.github/workflows/release.yml --- old/nltk-3.10.1/.github/workflows/release.yml 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/.github/workflows/release.yml 2026-08-05 10:16:23.000000000 +0200 @@ -17,6 +17,7 @@ uses: actions/checkout@v7 with: fetch-depth: 0 + fetch-tags: true - name: Check that release tag points to a develop commit run: | @@ -83,16 +84,19 @@ build-and-github-release: needs: gate-on-ci - name: Build and create GitHub draft release + name: Build and create GitHub release runs-on: ubuntu-latest permissions: contents: write steps: - name: Checkout repository uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: '3.12' @@ -104,9 +108,29 @@ - name: Build package run: python -m build + - name: Extract release notes from ChangeLog + run: | + set -euo pipefail + + RAW_VERSION=${GITHUB_REF_NAME#v} + VERSION=${RAW_VERSION%%-*} + + awk -v version="$VERSION" ' + $1 == "Version" && $2 == version { capture=1; found=1; print; next } + capture && $1 == "Version" { exit } + capture { print } + END { + if (!found) { + print "ERROR: Version " version " not found in ChangeLog" > "/dev/stderr" + exit 2 + } + } + ' ChangeLog > release-notes.md + + test -s release-notes.md + - name: Create GitHub Release uses: softprops/action-gh-release@v3 with: files: dist/* - draft: true - generate_release_notes: true + body_path: release-notes.md diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/ChangeLog new/nltk-3.10.2/ChangeLog --- old/nltk-3.10.1/ChangeLog 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/ChangeLog 2026-08-05 10:16:23.000000000 +0200 @@ -1,3 +1,14 @@ +Version 3.10.2 2026-08-05 + +* Remove inisec.py and document PYTHONSAFEPATH instead +* Skip draft step in release workflow +* Fix symlink escape in FramenetCorpusReader (CWE-59) +* Guard tempfile.gettempdir() when building pathsec allowed roots +* add tests for transitive_closure + +Thanks to the following contributors to 3.10.2: +Litesh Ghute, Eric Kafe, Evan Kiefer, tarann26 and Rav Singh Chandan + Version 3.10.1 2026-07-29 * Expand `~` in env-var paths diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/SECURITY.md new/nltk-3.10.2/SECURITY.md --- old/nltk-3.10.1/SECURITY.md 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/SECURITY.md 2026-08-05 10:16:23.000000000 +0200 @@ -81,6 +81,119 @@ This makes the trust decision explicit and avoids surprising behavior in server-side or shared execution environments. +### Module import hijacking (CWE-426) + +NLTK uses lazy (inline) imports for optional dependencies such as +`numpy`, `joblib`, and `tqdm`. Like any Python code, these imports are +resolved through `sys.path`. + +By default, Python prepends a path to `sys.path` at interpreter startup: +for `python script.py` it is the script's directory, and for +`python -m module`, `python -c ...`, or the REPL it is the current +working directory. If you run Python from an untrusted or +world-writable directory, an attacker who can place a malicious +`numpy.py`, `joblib.py`, or `tqdm.py` there can have it imported instead +of the real dependency, leading to arbitrary code execution. + +This is an **interpreter-level** search-path issue (CWE-426), not +specific to NLTK. It cannot be reliably fixed from within a library: +by the time `import nltk` runs, `sys.path` is already built and other +modules may already have been imported against the unsafe entry. Deciding +whether the current directory should be on `sys.path` is the host +application's prerogative, and Python provides a direct way to make that +choice. + +#### Recommended mitigation + +Start Python so the unsafe path is never added in the first place. On +Python 3.11+: + +```bash +python -P your_script.py +# or, per invocation: +PYTHONSAFEPATH=1 python your_script.py +``` + +`-P` / `PYTHONSAFEPATH` omit the auto-prepended script/CWD entry from +`sys.path` at startup, before any import runs. This applies to *all* +imports in the process, not just NLTK's, and is the mitigation CPython +itself recommends for untrusted working directories. + +#### Enabling it conveniently + +For a one-off command, prefix the invocation: + +```bash +PYTHONSAFEPATH=1 python your_script.py +``` + +To apply it to every Python process you launch, export the environment +variable from your shell profile (e.g. `~/.bashrc`, `~/.zshrc`, or +`~/.profile`): + +```bash +export PYTHONSAFEPATH=1 +``` + +Because it is an environment variable, it is inherited by scripts, +subprocesses, virtual-environment interpreters, cron jobs, and most +tools that launch Python — which the command-line `-P` flag is not. This +is the most reliable "set once" option. + +> **Note:** exporting `PYTHONSAFEPATH=1` changes behavior for *all* your +> Python programs. A few programs legitimately rely on importing modules +> from the current working directory (for example, running a script that +> imports a sibling file). If you hit an unexpected `ModuleNotFoundError` +> after enabling it, that program needs the CWD on `sys.path` and should +> be run without the flag, or its directory added to `PYTHONPATH` +> explicitly. + +A shell alias such as `alias python='python -P'` is possible but **not +recommended as a security control**: aliases apply only to interactive +shells and only to the exact command name `python`. They do not cover +shebang scripts, `python3`/`py`/venv interpreters, IDEs, notebook +kernels, or subprocesses, so they can leave gaps. Prefer the exported +environment variable. + +#### Enabling it in CI + +Setting `PYTHONSAFEPATH=1` in continuous integration keeps your test run +under the same policy recommended above. Set it in the environment of the +step(s) that execute your test suite, rather than globally, so that other +steps which legitimately import a package from the checked-out source tree +(for example, `python -c "import yourpkg; ..."` before an install) are not +broken by the stricter search path. This is not a substitute for CI +isolation: in CI the working directory is the checked-out repository +itself, so if that code is untrusted (for example, a fork pull request) +the real protection comes from the CI platform running it with restricted +permissions and no secrets — GitHub Actions does this for fork +`pull_request` runs — rather than from `-P`. + +NLTK's own CI sets `PYTHONSAFEPATH=1` on the pytest step, and the test +suite includes `test_safe_path_blocks_cwd_import`, which verifies that a +module in the current working directory is not importable under `-P`. + +#### Limitations + +`-P` / `PYTHONSAFEPATH` removes only the *automatic, implicit* prepending +of the script/CWD directory at interpreter startup. It does **not** +prevent code that runs later — the host application, a dependency, or +test tooling such as `pytest` — from deliberately re-adding the current +directory, e.g.: + +```python +import sys +sys.path.insert(0, "") # CWD is searchable again +``` + +It is therefore a strong, sensible default rather than an irreversible +sandbox. Its value is that the CWD is no longer searched *silently and by +default*; re-adding it afterward is an explicit act by code you already +trust to run in your process. NLTK does not attempt to enforce this from +within the library, because doing so would mean mutating the host +application's global `sys.path` — which is both easily undone and poor +library etiquette. + ### Network URL validation NLTK permits network resource loading only for `http:` and `https:` diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/VERSION new/nltk-3.10.2/nltk/VERSION --- old/nltk-3.10.1/nltk/VERSION 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/VERSION 2026-08-05 10:16:23.000000000 +0200 @@ -1 +1 @@ -3.10.1 +3.10.2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/__init__.py new/nltk-3.10.2/nltk/__init__.py --- old/nltk-3.10.1/nltk/__init__.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/__init__.py 2026-08-05 10:16:23.000000000 +0200 @@ -18,8 +18,6 @@ isort:skip_file """ -# Import security hook – must come first to protect against CWD hijacking -import nltk.inisec import os import importlib import sys diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/corpus/reader/framenet.py new/nltk-3.10.2/nltk/corpus/reader/framenet.py --- old/nltk-3.10.1/nltk/corpus/reader/framenet.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/corpus/reader/framenet.py 2026-08-05 10:16:23.000000000 +0200 @@ -779,6 +779,26 @@ raise FramenetError(f"Invalid {kind}: {value!r}") +def _validate_in_root(locpath, root, context): + """Reject a resolved path that escapes the corpus root through a symlink. + + ``_reject_unsafe_path_component`` only rejects unsafe characters in the + caller-/corpus-supplied name itself; it never resolves symlinks. A + symlink planted inside the corpus subdirectory (``frame/``, ``lu/`` or + ``fulltext/``) can still point outside the corpus root even when the name + referencing it contains no separator or ``..`` at all. + + This calls ``nltk.pathsec.validate_path`` with the corpus root as + ``required_root``, the same symlink-resolving containment guard used by + ``CorpusReader.open()`` and ``NKJPCorpusReader.add_root()``: both + ``locpath`` and ``root`` are resolved with ``Path.resolve()`` before the + containment check, so a symlink cannot escape undetected. + """ + from nltk.pathsec import validate_path + + validate_path(locpath, context=context, required_root=root) + + class AttrDict(dict): """A class that wraps a dict and allows accessing the keys of the dict as if they were attributes. Taken from here: @@ -1385,15 +1405,21 @@ except KeyError as e: # probably means that fn_docid was not in the index raise FramenetError(f"Unknown document id: {fn_docid}") from e - # Security (CWE-22): defend against a malicious corpus index whose - # filename field contains path-traversal sequences. Reject the unsafe - # name and resolve the path through self.abspath() so the file is read - # via the PathPointer / nltk.pathsec sandbox instead of the builtin - # open() that a bare string path would use in XMLCorpusView. + # Security (CWE-22 / CWE-59): defend against a malicious corpus index + # whose filename field contains path-traversal sequences or a symlink + # planted inside the fulltext directory. Reject the unsafe name, then + # resolve through self.abspath() and validate_path() with the corpus + # root as required_root -- the same symlink-resolving containment + # guard CorpusReader.open() and NKJPCorpusReader use -- so the file is + # read via the PathPointer / nltk.pathsec sandbox instead of the + # builtin open() that a bare string path would use in XMLCorpusView, + # and a symlink cannot escape the corpus root even though its own name + # contains no separators or "..". _reject_unsafe_path_component(xmlfname, "document filename") # construct the path name for the xml file containing the document info locpath = self.abspath(os.path.join(self._fulltext_dir, xmlfname)) + _validate_in_root(locpath, self.root, "FramenetCorpusReader") # Grab the top-level xml element containing the fulltext annotation with XMLCorpusView(locpath, "fullTextAnnotation") as view: @@ -1482,16 +1508,22 @@ elif not self._frame_idx: self._buildframeindex() - # Security (CWE-22): the frame name is interpolated into the XML file - # path. Reject crafted names, then resolve through self.abspath() so the - # file is read via the PathPointer / nltk.pathsec sandbox rather than the - # builtin open() that a bare string path would use in XMLCorpusView. + # Security (CWE-22 / CWE-59): the frame name is interpolated into the + # XML file path. Reject crafted names, then resolve through + # self.abspath() and validate_path() with the corpus root as + # required_root -- the same symlink-resolving containment guard + # CorpusReader.open() and NKJPCorpusReader use -- so the file is read + # via the PathPointer / nltk.pathsec sandbox rather than the builtin + # open() that a bare string path would use in XMLCorpusView, and a + # symlink planted inside the frame directory cannot escape the corpus + # root even though its own name contains no separators or "..". _reject_unsafe_path_component(fn_fname, "frame name") # construct the path name for the xml file containing the Frame info # Grab the xml for the frame try: locpath = self.abspath(os.path.join(self._frame_dir, fn_fname + ".xml")) + _validate_in_root(locpath, self.root, "FramenetCorpusReader") with XMLCorpusView(locpath, "frame") as view: elt = view[0] except OSError as e: @@ -1835,11 +1867,16 @@ """ fn_luid = lu.ID - # Security (CWE-22): the LU id comes from corpus data (a <lexUnit ID="..."> - # attribute) and is interpolated into the XML file path. A non-numeric - # id can carry path-traversal sequences; reject it, then resolve through - # self.abspath() so the file is read via the PathPointer / nltk.pathsec - # sandbox rather than the builtin open() used for a bare string path. + # Security (CWE-22 / CWE-59): the LU id comes from corpus data (a + # <lexUnit ID="..."> attribute) and is interpolated into the XML file + # path. A non-numeric id can carry path-traversal sequences; reject + # it, then resolve through self.abspath() and validate_path() with the + # corpus root as required_root -- the same symlink-resolving + # containment guard CorpusReader.open() and NKJPCorpusReader use -- + # so the file is read via the PathPointer / nltk.pathsec sandbox + # rather than the builtin open() used for a bare string path, and a + # symlink planted inside the LU directory cannot escape the corpus + # root even though its own name contains no separators or "..". _reject_unsafe_path_component(fn_luid, "LU id") fname = f"lu{fn_luid}.xml" @@ -1848,6 +1885,7 @@ try: locpath = self.abspath(os.path.join(self._lu_dir, fname)) + _validate_in_root(locpath, self.root, "FramenetCorpusReader") with XMLCorpusView(locpath, "lexUnit") as view: elt = view[0] except OSError as e: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/corpus/reader/ipipan.py new/nltk-3.10.2/nltk/corpus/reader/ipipan.py --- old/nltk-3.10.1/nltk/corpus/reader/ipipan.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/corpus/reader/ipipan.py 2026-08-05 10:16:23.000000000 +0200 @@ -187,6 +187,24 @@ return list(ret_fileids) def _get_tag(self, f, tag): + # Security (CWE-22 / CWE-59): ``f`` reaches this point as a plain str, + # not a PathPointer -- ``_list_header_files`` / ``_list_morph_files_by`` + # call ``.replace("morph.xml", "header.xml")`` on the result of + # ``self.abspath()``/``self.abspaths()``, and since + # ``FileSystemPathPointer`` subclasses ``str``, ``.replace()`` returns + # a plain string, silently discarding the PathPointer wrapper. That + # meant this ``open()`` call never went through nltk.pathsec at all, + # not even the global check that a PathPointer.open() would apply, so + # a symlink planted inside the corpus root (with a name containing no + # separators or "..", so it passes FileSystemPathPointer.join() + # cleanly) could point anywhere on the filesystem and still be + # opened. Validate the resolved path against the corpus root before + # opening, the same symlink-resolving containment guard + # CorpusReader.open() and NKJPCorpusReader use. + from nltk.pathsec import validate_path + + validate_path(f, context="IPIPANCorpusReader", required_root=self.root) + tags = [] with open(f) as infile: header = infile.read() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/inisec.py new/nltk-3.10.2/nltk/inisec.py --- old/nltk-3.10.1/nltk/inisec.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/inisec.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,157 +0,0 @@ -# Natural Language Toolkit: Security – early import hook -# -# Copyright (C) 2026 NLTK Project -# Author: Eric Kafe <[email protected]> -# URL: <https://www.nltk.org/> -# For license information, see LICENSE.TXT - -""" -Early security module to prevent module hijacking from the current working directory. - -This module installs a custom `MetaPathFinder` to mitigate Uncontrolled Search Path -vulnerabilities (CWE-427). It uses a dynamic, full-stack inspection architecture -to protect NLTK and its entire dependency tree without relying on a hardcoded list -of vulnerable modules, and without globally mutating the host application's -``sys.path``. - -The hook is installed at the very top of `nltk/__init__.py` to ensure it is active -before any NLTK code executes. - -Scope of protection --------------------- -* **Main process (synchronous imports).** The meta-path finder inspects the call - stack whenever a module is resolved from the current working directory (CWD). If - any ancestor frame belongs to ``nltk``, the import is blocked. This covers both - direct imports and indirect ones initiated by NLTK through a dependency - (e.g. ``nltk -> sklearn -> joblib``). The host application's own CWD imports are - left untouched. - -* **Freshly started worker interpreters (``spawn`` / ``forkserver``).** These read - ``PYTHONSAFEPATH`` during interpreter startup and therefore omit the CWD from - ``sys.path`` natively. NLTK propagates ``PYTHONSAFEPATH=1`` into the environment - at install time so such workers inherit interpreter-level isolation. - -Known residual limitation -------------------------- -Imports executed inside a **``fork``-based worker** (the default for some parallel -backends, e.g. joblib/loky on Linux) are *not* fully covered. A forked worker -inherits the parent's already-fixed ``sys.path`` (so ``PYTHONSAFEPATH`` is never -re-evaluated) and runs on a fresh call stack with no ``nltk`` frame (so caller -detection cannot fire). Closing this in-library would require stripping the CWD -from the *parent's* ``sys.path`` — a global mutation this module deliberately -avoids because it can break a host application's legitimate local imports. - -To obtain complete protection across fork-based workers, start Python with the -``-P`` flag or set ``PYTHONSAFEPATH=1`` in the environment **before** launching the -process. - -Note: ``-P`` and ``PYTHONSAFEPATH`` require **Python 3.11+**. On Python 3.10 they -are silently ignored, so this launch-time remedy is unavailable and fork-based -worker imports cannot be isolated this way. (Python 3.10 is expected to be dropped -in an upcoming release.) - -The hook can be disabled entirely by setting ``NLTK_DISABLE_IMPORT_SECURITY=1``. -""" - -import importlib.abc -import importlib.machinery -import os -import sys -from pathlib import Path - - -class NLTKSafeImportFinder(importlib.abc.MetaPathFinder): - """ - Custom finder that dynamically blocks NLTK and its dependencies from - importing modules from the current working directory. - """ - - def _is_import_from_nltk(self): - """ - Walk the *entire* call stack to determine if NLTK is an ancestor. - This correctly catches indirect imports (e.g., NLTK -> sklearn -> joblib) - that occur synchronously in the current process. - """ - try: - frame = sys._getframe(2) - while frame: - module_name = frame.f_globals.get("__name__") - if module_name: - if module_name.startswith("nltk") and module_name != "nltk.inisec": - return True - frame = frame.f_back - except Exception: - pass - return False - - def find_spec(self, fullname, path, target=None): - # 1. Exempt NLTK itself to ensure local development from repo root works. - top_level = fullname.split(".")[0] - if top_level == "nltk": - return None - - # 2. Ask default PathFinder where this module lives - spec = importlib.machinery.PathFinder.find_spec(fullname, path, target) - if spec is None: - return None - - # 3. Resolve CWD dynamically (respects process chdir) - try: - cwd = Path.cwd().resolve() - except FileNotFoundError: - return None - - # 4. Check if the module or package resolves to the CWD (or any subdirectory within it) - is_cwd = False - try: - if spec.origin: - resolved_origin = Path(spec.origin).resolve() - resolved_origin.relative_to(cwd) - is_cwd = True - elif spec.submodule_search_locations: - for loc in spec.submodule_search_locations: - resolved_loc = Path(loc).resolve() - resolved_loc.relative_to(cwd) - is_cwd = True - break - except (ValueError, FileNotFoundError): - # ValueError means the path is NOT inside cwd; FileNotFoundError means it doesn't exist - pass - - # 5. FAST EXIT: If the module is NOT in the CWD, let it load instantly. - if not is_cwd: - return None - - # 6. It is in the CWD. Block only if NLTK initiated this import - # (directly or indirectly) in the current process. - if self._is_import_from_nltk(): - raise ImportError( - f"Blocked import of {fullname} from current working directory " - "for security reasons. Use '-P' or set PYTHONSAFEPATH to prevent " - "Python from searching the current working directory." - ) - - # 7. CWD import requested by the host application (not NLTK), allow it. - return None - - -def _install(): - """ - Install the finder once and propagate interpreter-level CWD isolation to - freshly started worker interpreters (``spawn`` / ``forkserver``) via - ``PYTHONSAFEPATH``. - - ``setdefault`` is used so NLTK never overrides a value the host has already - chosen; it only supplies a default for child interpreters to inherit. Note - that setting this has no effect on the *current* interpreter (the flag is - read only at startup) — it exists purely for inheritance by children. - """ - os.environ.setdefault("PYTHONSAFEPATH", "1") - - if not any(isinstance(f, NLTKSafeImportFinder) for f in sys.meta_path): - sys.meta_path.insert(0, NLTKSafeImportFinder()) - - -# Install the finder only once, unless explicitly disabled via environment variable -if os.environ.get("NLTK_DISABLE_IMPORT_SECURITY") != "1": - _install() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/pathsec.py new/nltk-3.10.2/nltk/pathsec.py --- old/nltk-3.10.1/nltk/pathsec.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/pathsec.py 2026-08-05 10:16:23.000000000 +0200 @@ -56,7 +56,13 @@ import tempfile - for loc in ["~/nltk_data", "/usr/share/nltk_data", tempfile.gettempdir()]: + candidate_locs = ["~/nltk_data", "/usr/share/nltk_data"] + try: + candidate_locs.append(tempfile.gettempdir()) + except (OSError, ValueError, RuntimeError): + pass + + for loc in candidate_locs: try: p = Path(loc).expanduser().resolve() if p.exists(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_framenet_security.py new/nltk-3.10.2/nltk/test/unit/test_framenet_security.py --- old/nltk-3.10.1/nltk/test/unit/test_framenet_security.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/test/unit/test_framenet_security.py 2026-08-05 10:16:23.000000000 +0200 @@ -1,4 +1,4 @@ -"""Regression tests for path traversal in FramenetCorpusReader (CWE-22). +"""Regression tests for path traversal in FramenetCorpusReader (CWE-22 / CWE-59). ``doc()``, ``frame_by_name()`` and ``_lu_file()`` interpolate a caller- or corpus-supplied name into an XML file path that is then read via @@ -9,6 +9,13 @@ Paths are built with ``os.path.join`` / ``os.pardir`` so the tests behave the same on POSIX and Windows. + +A separate class of tests below covers a symlink planted inside the corpus +subdirectory itself (``frame/``, ``lu/`` or ``fulltext/``). Its own name +contains no separator or ``..``, so it passes ``_reject_unsafe_path_component`` +cleanly; only resolving the path with ``Path.resolve()`` and checking it +against the corpus root (what ``_validate_in_root`` / ``validate_path`` do) +catches this. """ import builtins @@ -158,3 +165,78 @@ @pytest.mark.parametrize("ok", ["TestFrame", "Apply_heat", "lu123", "a.b-c"]) def test_reject_unsafe_path_component_allows_normal(ok): _reject_unsafe_path_component(ok, "frame name") # must not raise + + +# --- symlink escape: name has no separator or ".." but still leaves the root -- + + +def _symlink_or_skip(target, link): + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not supported in this environment") + + +def test_framenet_frame_rejects_symlink_escape(tmp_path, monkeypatch): + """A symlink inside frame/ passes the name guard but must still be blocked.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "pwn.xml" + secret.write_text(_FRAME_XML.format(name="pwned")) + + link = root / "frame" / "evil_link.xml" + _symlink_or_skip(secret, link) + + fn = FramenetCorpusReader(str(root), []) + opened = _record_opens(monkeypatch) + with pytest.raises(ValueError, match="escapes root"): + fn.frame("evil_link") + assert not any( + "outside" in p for p in opened + ), "symlink escape reached the filesystem" + + +def test_framenet_lu_file_rejects_symlink_escape(tmp_path, monkeypatch): + """A symlink inside lu/ passes the id guard but must still be blocked.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "pwn.xml" + secret.write_text( + '<?xml version="1.0"?><lexUnit ID="1" name="pwned.n" status="Created"/>' + ) + + link = root / "lu" / "luevilLU.xml" + _symlink_or_skip(secret, link) + + fn = FramenetCorpusReader(str(root), []) + fn._lu_idx = {"__dummy__": AttrDict({"name": "__dummy__"})} # skip _buildluindex() + lu = AttrDict({"ID": "evilLU"}) + opened = _record_opens(monkeypatch) + with pytest.raises(ValueError, match="escapes root"): + fn._lu_file(lu) + assert not any( + "outside" in p for p in opened + ), "symlink escape reached the filesystem" + + +def test_framenet_doc_rejects_symlink_escape(tmp_path, monkeypatch): + """A symlink inside fulltext/ passes the filename guard but must still be blocked.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "pwn.xml" + secret.write_text('<?xml version="1.0"?><fullTextAnnotation></fullTextAnnotation>') + + link = root / "fulltext" / "evilDoc.xml" + _symlink_or_skip(secret, link) + + fn = FramenetCorpusReader(str(root), []) + fn._fulltext_idx = {7: AttrDict({"filename": "evilDoc.xml"})} + opened = _record_opens(monkeypatch) + with pytest.raises(ValueError, match="escapes root"): + fn.doc(7) + assert not any( + "outside" in p for p in opened + ), "symlink escape reached the filesystem" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_ipipan_security.py new/nltk-3.10.2/nltk/test/unit/test_ipipan_security.py --- old/nltk-3.10.1/nltk/test/unit/test_ipipan_security.py 1970-01-01 01:00:00.000000000 +0100 +++ new/nltk-3.10.2/nltk/test/unit/test_ipipan_security.py 2026-08-05 10:16:23.000000000 +0200 @@ -0,0 +1,129 @@ +"""Regression tests for symlink-based path escape in IPIPANCorpusReader +(CWE-22 / CWE-59). + +``channels()``, ``domains()``, ``categories()`` and ``fileids(channels=...)`` +route through ``_get_tag()``, which used to call the builtin ``open()`` on a +plain string produced by ``.replace("morph.xml", "header.xml")`` on the +result of ``self.abspath()``. Since ``FileSystemPathPointer`` subclasses +``str``, that ``.replace()`` call silently discarded the ``PathPointer`` +wrapper, so the resulting ``open()`` never went through ``nltk.pathsec`` at +all, not even the global, non-scoped check. A symlink planted inside the +corpus root, with a name containing no separators or "..", passed the +existing traversal guard cleanly and could point anywhere on the filesystem. + +Paths are built with ``os.path.join`` / ``os.pardir`` so the tests behave the +same on POSIX and Windows. +""" + +import os + +import pytest + +from nltk.corpus.reader.ipipan import IPIPANCorpusReader + + +def _make_corpus(tmp_path): + root = tmp_path / "ipipan" + root.mkdir() + (root / "real_morph.xml").write_text("<tei>legit morph</tei>") + (root / "real_header.xml").write_text("<channel>legit-channel</channel>") + return root + + +def _symlink_or_skip(target, link): + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not supported in this environment") + + +def test_ipipan_channels_reads_legitimate_file(tmp_path): + """A real in-root file must still work after adding the containment check.""" + root = _make_corpus(tmp_path) + reader = IPIPANCorpusReader(str(root), r".*\.xml") + assert reader.channels(fileids=["real_morph.xml"]) == ["legit-channel"] + + +def test_ipipan_channels_rejects_traversal_fileid(tmp_path): + """A ../ traversal fileid is rejected before any file is opened.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "pwn_header.xml").write_text("<channel>pwned</channel>") + + reader = IPIPANCorpusReader(str(root), r".*\.xml") + evil = os.path.join(os.pardir, "outside", "pwn_morph.xml") + with pytest.raises(ValueError, match="Traversal blocked"): + reader.channels(fileids=[evil]) + + +def test_ipipan_channels_rejects_symlink_escape(tmp_path): + """A symlink inside the corpus root passes the name guard but must still + be blocked, even though its own name has no separators or "..".""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "stolen.xml" + secret.write_text("<channel>TOP-SECRET-DATA-OUTSIDE-CORPUS-ROOT</channel>") + + link = root / "evil_link.xml" + _symlink_or_skip(secret, link) + + # Constructed the normal way: fileids is a regex, so the reader + # auto-discovers the symlink exactly as it would a real file. + reader = IPIPANCorpusReader(str(root), r".*\.xml") + assert "evil_link.xml" in reader.fileids() + + with pytest.raises(ValueError, match="escapes root"): + reader.channels(fileids=["evil_link.xml"]) + + +def test_ipipan_domains_rejects_symlink_escape(tmp_path): + """domains() shares _get_tag() with channels(); confirm it is covered too.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "stolen.xml" + secret.write_text("<domain>pwned</domain>") + + link = root / "evil_link2.xml" + _symlink_or_skip(secret, link) + + reader = IPIPANCorpusReader(str(root), r".*\.xml") + with pytest.raises(ValueError, match="escapes root"): + reader.domains(fileids=["evil_link2.xml"]) + + +def test_ipipan_categories_rejects_symlink_escape(tmp_path): + """categories() also shares _get_tag() with channels(); confirm it is + covered too. It post-processes _parse_header()'s result through + _map_category(), but the ValueError must fire before that ever runs.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "stolen.xml" + secret.write_text("<keyTerm>pwned</keyTerm>") + + link = root / "evil_link4.xml" + _symlink_or_skip(secret, link) + + reader = IPIPANCorpusReader(str(root), r".*\.xml") + with pytest.raises(ValueError, match="escapes root"): + reader.categories(fileids=["evil_link4.xml"]) + + +def test_ipipan_fileids_by_channel_rejects_symlink_escape(tmp_path): + """_list_morph_files_by() (used by fileids(channels=...)) shares the same + _get_tag() call and must be covered too.""" + root = _make_corpus(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "stolen.xml" + secret.write_text("<channel>pwned</channel>") + + link = root / "evil_link3.xml" + _symlink_or_skip(secret, link) + + reader = IPIPANCorpusReader(str(root), r".*\.xml") + with pytest.raises(ValueError, match="escapes root"): + reader.fileids(channels=["prasa"]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_pathsec.py new/nltk-3.10.2/nltk/test/unit/test_pathsec.py --- old/nltk-3.10.1/nltk/test/unit/test_pathsec.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/test/unit/test_pathsec.py 2026-08-05 10:16:23.000000000 +0200 @@ -144,6 +144,39 @@ pathsec.open(outside, "r") +# --- ALLOWED-ROOTS / TEMP-DIR FALLBACK TESTS --- + + +def test_get_allowed_roots_survives_missing_tempdir(tmp_path, monkeypatch): + """Regression test for issue #3716. + + ``_get_allowed_roots()`` used to build its fallback-location list as a + literal ``[..., tempfile.gettempdir()]``, which evaluates + ``tempfile.gettempdir()`` while constructing the list -- *before* the + loop's ``try/except`` runs. On a system with no usable temp directory + (read-only root filesystem, nothing mounted at ``/tmp``), ``gettempdir()`` + raises ``FileNotFoundError`` (an ``OSError`` subclass) that propagates out + of the whole function, discarding the roots already collected from + ``nltk.data.path``/``NLTK_DATA`` and breaking resource lookups (e.g. + ``sent_tokenize()``) even when the resource is already cached. + """ + import nltk.data + + known_root = tmp_path / "nltk_data_known_root" + known_root.mkdir() + + monkeypatch.setattr(nltk.data, "path", nltk.data.path + [str(known_root)]) + + # Force a clean cache: _get_allowed_roots() memoizes on (data.path, NLTK_DATA). + pathsec._ALLOWED_ROOTS_CACHE = None + pathsec._LAST_DATA_PATHS = None + + with patch("tempfile.gettempdir", side_effect=FileNotFoundError("no temp dir")): + roots = pathsec._get_allowed_roots() + + assert known_root.resolve() in roots + + # --- ZIP-SLIP TESTS --- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_security.py new/nltk-3.10.2/nltk/test/unit/test_security.py --- old/nltk-3.10.1/nltk/test/unit/test_security.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/test/unit/test_security.py 2026-08-05 10:16:23.000000000 +0200 @@ -1,275 +1,45 @@ -import importlib.util import os import subprocess import sys import tempfile -from pathlib import Path -from unittest import mock import pytest -def test_module_hijacking_prevention(): - """Ensure imports of vulnerable modules from CWD are blocked.""" - parent_paths = [p for p in sys.path if p and p != "."] - - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "joblib.py"), "w") as f: - f.write("print('HIJACK_SUCCESS')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import sys\n" - f"sys.path = {repr(parent_paths)} + sys.path\n" - "from nltk.util import parallelize_preprocess\n" - "list(parallelize_preprocess(str.upper, ['a'], processes=1))\n" - ) - env = os.environ.copy() - res = subprocess.run( - [sys.executable, "victim.py"], - cwd=d, - env=env, - capture_output=True, - text=True, - ) - assert "HIJACK_SUCCESS" not in res.stdout - - -def test_host_imports_of_vulnerable_modules_are_blocked(): - """CWD imports pulled in during `import nltk` are blocked.""" - parent_paths = [p for p in sys.path if p and p != "."] - - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "joblib.py"), "w") as f: - f.write("print('HOST_HIJACK')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import sys\n" - f"sys.path = {repr(parent_paths)} + sys.path\n" - "import nltk\n" - "import joblib\n" - "print('HOST_SUCCESS')\n" - ) - env = os.environ.copy() - res = subprocess.run( - [sys.executable, "victim.py"], - cwd=d, - env=env, - capture_output=True, - text=True, - ) - assert "HOST_HIJACK" not in res.stdout - - -def test_host_imports_of_non_vulnerable_modules_are_unaffected(): - """Host imports of non‑vulnerable modules from CWD succeed.""" - parent_paths = [p for p in sys.path if p and p != "."] - - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "antigravity.py"), "w") as f: - f.write("print('ANTIGRAVITY_HIJACK')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import sys\n" - f"sys.path = [''] + {repr(parent_paths)} + sys.path\n" - "import nltk\n" - "import antigravity\n" - "print('HOST_SUCCESS')\n" - ) - env = os.environ.copy() - env.pop("PYTHONSAFEPATH", None) # this test needs the CWD searchable - res = subprocess.run( - [sys.executable, "victim.py"], - cwd=d, - env=env, - capture_output=True, - text=True, - ) - try: - assert res.returncode == 0 - assert "ANTIGRAVITY_HIJACK" in res.stdout - assert "HOST_SUCCESS" in res.stdout - except AssertionError: - print("--- STDOUT ---\n", res.stdout) - print("--- STDERR ---\n", res.stderr) - raise - - -def test_disable_flag(): - """Ensure setting NLTK_DISABLE_IMPORT_SECURITY=1 disables the hook.""" - parent_paths = [p for p in sys.path if p and p != "."] - - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "joblib.py"), "w") as f: - f.write("print('DISABLED_HIJACK')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import os\n" - f"os.environ['NLTK_DISABLE_IMPORT_SECURITY'] = '1'\n" - f"import sys\n" - f"sys.path = [''] + {repr(parent_paths)} + sys.path\n" - "import nltk\n" - "import joblib\n" - "print('SUCCESS')\n" - ) - env = os.environ.copy() - env.pop("PYTHONSAFEPATH", None) # must be able to reach the CWD module - res = subprocess.run( - [sys.executable, "victim.py"], - cwd=d, - env=env, - capture_output=True, - text=True, - ) - try: - assert ( - "DISABLED_HIJACK" in res.stdout - ), "Malicious module should be loaded when hook is disabled" - except AssertionError: - print("--- STDOUT ---\n", res.stdout) - print("--- STDERR ---\n", res.stderr) - raise - - @pytest.mark.skipif( - not importlib.util.find_spec("joblib"), reason="joblib not installed" + sys.version_info < (3, 11), + reason="PYTHONSAFEPATH / -P exist only on Python 3.11+.", ) -def test_legitimate_import_from_site_packages(): - import joblib - - assert joblib.__file__ is not None - - -def test_pythonsafepath_is_propagated_for_child_interpreters(): - """ - Installing the hook must set PYTHONSAFEPATH=1 so freshly started worker - interpreters (spawn/forkserver) inherit interpreter-level CWD isolation. +def test_safe_path_blocks_cwd_import(): """ - from nltk import inisec - - with mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop("PYTHONSAFEPATH", None) - inisec._install() - assert os.environ.get("PYTHONSAFEPATH") == "1" - + A module placed in the current working directory must NOT be importable + when the interpreter is started with -P / PYTHONSAFEPATH, mitigating CWD + module hijacking (CWE-426). See SECURITY.md; NLTK's CI runs pytest under + this policy. -def test_pythonsafepath_does_not_override_host_choice(): - """ - NLTK must not override a PYTHONSAFEPATH value the host has already set. - """ - from nltk import inisec - - with mock.patch.dict(os.environ, {"PYTHONSAFEPATH": "0"}, clear=False): - inisec._install() - assert os.environ.get("PYTHONSAFEPATH") == "0" - - [email protected]( - strict=True, - reason=( - "Known residual limitation: fork-based joblib/loky workers inherit the " - "parent's already-fixed sys.path and run on a fresh stack, so neither " - "PYTHONSAFEPATH nor caller detection applies. Closing this in-library " - "would require globally mutating the parent's sys.path, which this " - "design deliberately avoids. Remedy: launch with -P / PYTHONSAFEPATH. " - "If this test starts passing, the limitation was closed and the xfail " - "must be removed." - ), -) [email protected]( - not importlib.util.find_spec("joblib"), reason="joblib not installed" -) -def test_worker_process_cwd_import_is_blocked(): + Launches a child interpreter with -P whose CWD contains a decoy module and + asserts the decoy cannot be imported, checking the specific + ModuleNotFoundError rather than merely a non-zero exit. """ - Documents the fork-worker residual: a module dropped in the CWD is imported - by a joblib/loky worker because the worker's fresh stack has no NLTK frame - and its inherited sys.path still contains the CWD. Expected to xfail under - the current in-library-only design. - """ - parent_paths = [p for p in sys.path if p and p != "."] - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "joblib_victim.py"), "w") as f: - f.write("print('WORKER_HIJACK')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import sys\n" - f"sys.path = {repr(parent_paths)} + sys.path\n" - "import nltk\n" - "from joblib import Parallel, delayed\n" - "\n" - "def work(_):\n" - " try:\n" - " import joblib_victim\n" - " return 'IMPORTED'\n" - " except ImportError:\n" - " return 'BLOCKED'\n" - "\n" - "if __name__ == '__main__':\n" - " out = Parallel(n_jobs=2)(delayed(work)(i) for i in range(2))\n" - " print('RESULTS', out)\n" - ) - env = os.environ.copy() - # Deliberately do NOT set PYTHONSAFEPATH here: this test characterises - # the residual that exists WITHOUT the recommended launch-time remedy. - env.pop("PYTHONSAFEPATH", None) - res = subprocess.run( - [sys.executable, "victim.py"], - cwd=d, - env=env, - capture_output=True, - text=True, - ) - assert "WORKER_HIJACK" not in res.stdout - assert "IMPORTED" not in res.stdout + with open(os.path.join(d, "cwd_decoy.py"), "w") as f: + f.write("raise AssertionError('decoy should not be imported')\n") - [email protected]( - sys.version_info < (3, 11), - reason="PYTHONSAFEPATH was added in Python 3.11; on earlier versions it is " - "silently ignored, so this launch-time remedy does not apply.", -) -def test_worker_process_is_protected_with_pythonsafepath(): - """ - Confirms the recommended remedy works: with PYTHONSAFEPATH=1 in the launch - environment, worker interpreters omit the CWD and the hijack is prevented. - Requires Python 3.11+, where PYTHONSAFEPATH exists. - """ - if not importlib.util.find_spec("joblib"): - pytest.skip("joblib not installed") - - parent_paths = [p for p in sys.path if p and p != "."] - - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "joblib_victim.py"), "w") as f: - f.write("print('WORKER_HIJACK')\n") - with open(os.path.join(d, "victim.py"), "w") as f: - f.write( - f"import sys\n" - f"sys.path = {repr(parent_paths)} + sys.path\n" - "import nltk\n" - "from joblib import Parallel, delayed\n" - "\n" - "def work(_):\n" - " try:\n" - " import joblib_victim\n" - " return 'IMPORTED'\n" - " except ImportError:\n" - " return 'BLOCKED'\n" - "\n" - "if __name__ == '__main__':\n" - " out = Parallel(n_jobs=2)(delayed(work)(i) for i in range(2))\n" - " print('RESULTS', out)\n" - ) - env = os.environ.copy() - env["PYTHONSAFEPATH"] = "1" res = subprocess.run( - [sys.executable, "victim.py"], + [sys.executable, "-P", "-c", "import cwd_decoy"], cwd=d, - env=env, capture_output=True, text=True, ) - assert "WORKER_HIJACK" not in res.stdout + + assert res.returncode != 0, ( + f"decoy in CWD was importable under -P.\n" + f"stdout: {res.stdout!r}\nstderr: {res.stderr!r}" + ) + assert ( + "ModuleNotFoundError" in res.stderr + ), f"expected ModuleNotFoundError, got:\nstderr: {res.stderr!r}" + assert "cwd_decoy" in res.stderr def test_wordnet_app_reference_decode_rejects_wrong_types(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_tokenize.py new/nltk-3.10.2/nltk/test/unit/test_tokenize.py --- old/nltk-3.10.1/nltk/test/unit/test_tokenize.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/test/unit/test_tokenize.py 2026-08-05 10:16:23.000000000 +0200 @@ -998,6 +998,47 @@ def test_sent_tokenize(self, sentences: str, expected: list[str]): assert sent_tokenize(sentences) == expected + def test_sent_tokenize_curly_quotes_and_guillemets_issue_2333(self): + # Regression test for https://github.com/nltk/nltk/issues/2333. + # + # A sentence-final Unicode curly closing quote or guillemet that + # directly abuts the period, with no intervening whitespace, used to + # hide the sentence boundary from Punkt entirely: the period matched + # neither the "non-word char" branch nor the "whitespace" branch of + # the period-context regex, so period_context_re() never even + # yielded a match at that position. NLTKWordTokenizer's + # STARTING_QUOTES / ENDING_QUOTES (see + # test_word_tokenize_opening_single_quote_padding above) already + # special-cased this character set; PunktLanguageVars did not. + assert sent_tokenize("“First sentence.” Next one.") == [ + "“First sentence.”", + "Next one.", + ] + assert sent_tokenize("He said “this is great.” Then left.") == [ + "He said “this is great.”", + "Then left.", + ] + assert sent_tokenize("She asked, “Are you coming?” He nodded.") == [ + "She asked, “Are you coming?”", + "He nodded.", + ] + assert sent_tokenize("Il a dit «bonjour.» Puis il est parti.") == [ + "Il a dit «bonjour.»", + "Puis il est parti.", + ] + + # Regression guards: straight quotes and plain text (no quotes at + # all) must tokenize exactly as before, unaffected by the new + # boundary-realignment / non-word-char classes. + assert sent_tokenize('He said "hi." Then left.') == [ + 'He said "hi."', + "Then left.", + ] + assert sent_tokenize("This is one sentence. This is another.") == [ + "This is one sentence.", + "This is another.", + ] + def test_string_tokenizer(self) -> None: sentence = "Hello there" tokenizer = CharTokenizer() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/test/unit/test_util.py new/nltk-3.10.2/nltk/test/unit/test_util.py --- old/nltk-3.10.1/nltk/test/unit/test_util.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/test/unit/test_util.py 2026-08-05 10:16:23.000000000 +0200 @@ -1,6 +1,6 @@ import pytest -from nltk.util import everygrams +from nltk.util import everygrams, transitive_closure @pytest.fixture @@ -80,3 +80,38 @@ ] output = list(everygrams(everygram_input, max_len=3, pad_left=True)) assert output == expected_output + + +def test_transitive_closure_chain(): + graph = {"a": {"b"}, "b": {"c"}, "c": set()} + expected = {"a": {"b", "c"}, "b": {"c"}, "c": set()} + assert transitive_closure(graph) == expected + + +def test_transitive_closure_reflexive(): + graph = {"a": {"b"}, "b": {"c"}, "c": set()} + expected = {"a": {"a", "b", "c"}, "b": {"b", "c"}, "c": {"c"}} + assert transitive_closure(graph, reflexive=True) == expected + + +def test_transitive_closure_cycle(): + graph = {"a": {"b"}, "b": {"a"}} + expected = {"a": {"a", "b"}, "b": {"a", "b"}} + assert transitive_closure(graph) == expected + + +def test_transitive_closure_empty(): + assert transitive_closure({}) == {} + + +def test_transitive_closure_does_not_mutate_input(): + graph = {"a": {"b"}, "b": set()} + original_sets = {k: v for k, v in graph.items()} + snapshot = {k: set(v) for k, v in graph.items()} + + transitive_closure(graph) + + assert graph == snapshot + assert set(graph) == set(snapshot) + # the same set objects, not new-but-equal ones + assert all(graph[k] is original_sets[k] for k in snapshot) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/nltk-3.10.1/nltk/tokenize/punkt.py new/nltk-3.10.2/nltk/tokenize/punkt.py --- old/nltk-3.10.1/nltk/tokenize/punkt.py 2026-07-31 06:03:21.000000000 +0200 +++ new/nltk-3.10.2/nltk/tokenize/punkt.py 2026-08-05 10:16:23.000000000 +0200 @@ -234,7 +234,15 @@ """sentence internal punctuation, which indicates an abbreviation if preceded by a period-final token.""" - re_boundary_realignment = re.compile(r'["\')\]}]+?(?:\s+|(?=--)|$)', re.MULTILINE) + # Treat the Unicode curly quotes (u'\u2018' u'\u2019' u'\u201c' u'\u201d') + # and guillemets (u'\xab' u'\xbb') as closing punctuation like the ASCII + # quotes, so a sentence-final curly/guillemet quote is realigned onto the + # sentence it follows. NLTKWordTokenizer (nltk/tokenize/destructive.py) + # already handles this same set (STARTING_QUOTES / ENDING_QUOTES, gh-1682). + re_boundary_realignment = re.compile( + r'["\')\]}\u2018\u2019\u201c\u201d\xab\xbb]+?(?:\s+|(?=--)|$)', + re.MULTILINE, + ) """Used to realign punctuation that should be included in a sentence although it follows the period (or ?, !).""" @@ -243,8 +251,11 @@ @property def _re_non_word_chars(self): - return r"(?:[)\";}\]\*:@\'\({\[%s])" % re.escape( - "".join(set(self.sent_end_chars) - {"."}) + # Including the curly quotes/guillemets here makes a period that + # directly abuts one still register as a sentence boundary. + return ( + r"(?:[)\";}\]\*:@\'\({\[\u2018\u2019\u201c\u201d\xab\xbb%s])" + % re.escape("".join(set(self.sent_end_chars) - {"."})) ) """Characters that cannot appear within words""" ++++++ nltk.obsinfo ++++++ --- /var/tmp/diff_new_pack.u7wiQN/_old 2026-08-13 13:17:00.176496307 +0200 +++ /var/tmp/diff_new_pack.u7wiQN/_new 2026-08-13 13:17:00.192497034 +0200 @@ -1,5 +1,5 @@ name: nltk -version: 3.10.1 -mtime: 1785470601 -commit: 35813c85af3f13d4f7196085854eafb3d0c5db02 +version: 3.10.2 +mtime: 1785917783 +commit: 474af1f5a94b1b8d53fc2b6defec3a2ce7633b74
