This is an automated email from the ASF dual-hosted git repository. shuke987 pushed a commit to branch codex/fix-pythonudf-nlp-jieba-master in repository https://gitbox.apache.org/repos/asf/doris.git
commit d04b5712981ceb32296be17a5078742855decc46 Author: shuke <[email protected]> AuthorDate: Fri Jul 17 10:08:33 2026 +0800 [fix](regression) Split Python NLP UDF archive --- ...udf_complex_zip.py => build_nlp_chinese_zip.py} | 41 ++++++++++++++------- .../py_udf_complex_scripts/jieba.LICENSE | 20 ---------- .../py_udf_complex_scripts/py_udf_complex.zip | Bin 5254713 -> 18349 bytes .../test_python_udf_nlp_chinese.groovy | 34 ++++++++++++++++- 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_py_udf_complex_zip.py b/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_nlp_chinese_zip.py similarity index 76% rename from regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_py_udf_complex_zip.py rename to regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_nlp_chinese_zip.py index dfee82819e2..860335ee41f 100644 --- a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_py_udf_complex_zip.py +++ b/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/build_nlp_chinese_zip.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -"""Build the Python complex-UDF archive with its jieba dependency.""" +"""Build the NLP-only Python UDF archive with its jieba dependency.""" import argparse import hashlib @@ -28,13 +28,6 @@ import zipfile JIEBA_VERSION = "0.42.1" JIEBA_ARCHIVE_SHA256 = "055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2" -UDF_SOURCES = ( - "business_logic.py", - "complex_udaf.py", - "complex_udtf.py", - "external_api.py", - "nlp_chinese.py", -) REQUIRED_JIEBA_FILES = ( "jieba/__init__.py", "jieba/dict.txt", @@ -42,12 +35,33 @@ REQUIRED_JIEBA_FILES = ( "jieba/posseg/__init__.py", ) ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +JIEBA_LICENSE = """The MIT License (MIT) + +Copyright (c) 2013 Sun Junyi + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" def parse_args(): script_dir = Path(__file__).resolve().parent parser = argparse.ArgumentParser( - description="Build py_udf_complex.zip from a verified jieba source archive" + description="Build nlp_chinese.zip from a verified jieba source archive" ) parser.add_argument( "--jieba-archive", @@ -57,7 +71,7 @@ def parse_args(): ) parser.add_argument( "--output", - default=script_dir / "py_udf_complex.zip", + default=script_dir / "nlp_chinese.zip", type=Path, help="output zip path", ) @@ -80,10 +94,7 @@ def load_entries(script_dir, jieba_archive): f"expected {JIEBA_ARCHIVE_SHA256}, got {actual_sha256}" ) - entries = {name: (script_dir / name).read_bytes() for name in UDF_SOURCES} - entries["THIRD_PARTY_LICENSES/jieba.txt"] = ( - script_dir / "jieba.LICENSE" - ).read_bytes() + entries = {"nlp_chinese.py": (script_dir / "nlp_chinese.py").read_bytes()} archive_prefix = PurePosixPath(f"jieba-{JIEBA_VERSION}") / "jieba" with tarfile.open(jieba_archive, "r:gz") as archive: @@ -107,6 +118,8 @@ def load_entries(script_dir, jieba_archive): raise ValueError(f"cannot read {member.name} from jieba archive") entries[str(PurePosixPath("jieba") / relative_path)] = extracted.read() + entries["THIRD_PARTY_LICENSES/jieba.txt"] = JIEBA_LICENSE.encode("utf-8") + missing = [name for name in REQUIRED_JIEBA_FILES if name not in entries] if missing: raise ValueError(f"jieba archive is missing required files: {missing}") diff --git a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/jieba.LICENSE b/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/jieba.LICENSE deleted file mode 100644 index 8333c508228..00000000000 --- a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/jieba.LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Sun Junyi - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/py_udf_complex.zip b/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/py_udf_complex.zip index 189e7b49bbd..f59ca05756f 100644 Binary files a/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/py_udf_complex.zip and b/regression-test/suites/pythonudf_complex_p0/py_udf_complex_scripts/py_udf_complex.zip differ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy index 5ebd44f0551..85d0ce65067 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy @@ -15,11 +15,43 @@ // specific language governing permissions and limitations // under the License. +import java.security.MessageDigest + suite("test_python_udf_nlp_chinese") { // Test Chinese NLP processing using jieba library // Dependencies: jieba - def pyPath = """${context.file.parent}/py_udf_complex_scripts/py_udf_complex.zip""" + def archiveName = "nlp_chinese_jieba_0.42.1_392d235b.zip" + def expectedSha256 = "392d235bf3249633f4a1749c6589f6bc8e3c6427e19457ae06e476f0b6a80b96" + def pyPath = """${context.file.parent}/py_udf_complex_scripts/${archiveName}""" + def archiveUrl = """${getS3Url()}/regression/pythonudf_complex_p0/${archiveName}""" + def archiveFile = new File(pyPath) + + def fileSha256 = { File file -> + MessageDigest digest = MessageDigest.getInstance("SHA-256") + file.withInputStream { input -> + byte[] buffer = new byte[8192] + int bytesRead + while ((bytesRead = input.read(buffer)) != -1) { + digest.update(buffer, 0, bytesRead) + } + } + return digest.digest().collect { String.format("%02x", it & 0xff) }.join() + } + + if (!archiveFile.exists() || fileSha256(archiveFile) != expectedSha256) { + logger.info("Downloading NLP Python zip from ${archiveUrl}".toString()) + def command = ["/usr/bin/curl", "-fL", "--retry", "3", "--retry-delay", "2", archiveUrl, "--output", pyPath] + def process = new ProcessBuilder(command).redirectErrorStream(true).start() + def output = process.inputStream.text + def code = process.waitFor() + if (code != 0) { + logger.info("NLP Python zip download output: ${output}".toString()) + } + assertEquals(0, code) + } + assertEquals(expectedSha256, fileSha256(archiveFile)) + scp_udf_file_to_all_be(pyPath) def runtime_version = getPythonUdfRuntimeVersion() log.info("Python zip path: ${pyPath}".toString()) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
