This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new ddb8a7d2f3 TIKA-4927: replace groovy ModelGetter with a best-effort
antrun fetch (#3246)
ddb8a7d2f3 is described below
commit ddb8a7d2f314c118de1ac37bb72230938c0009f7
Author: Tim Allison <[email protected]>
AuthorDate: Thu Sep 24 13:38:13 2026 -0400
TIKA-4927: replace groovy ModelGetter with a best-effort antrun fetch
(#3246)
---
tika-parent/pom.xml | 1 -
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 90 ++++++++--------
.../tika/parser/ner/NamedEntityParserTest.java | 30 ++++--
.../tika/parser/ner/nltk/NLTKNERecogniserTest.java | 6 ++
.../parser/ner/regex/RegexNERecogniserTest.java | 6 ++
.../tika/parser/ner/opennlp/ModelGetter.groovy | 119 ---------------------
.../apache/tika/parser/ner/opennlp/get-models.sh | 26 -----
7 files changed, 81 insertions(+), 197 deletions(-)
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 4ddff01817..a9d5770960 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -294,7 +294,6 @@
https://github.com/mojohaus/versions/issues/1070 -->
<checkstyle.plugin.version>3.6.0</checkstyle.plugin.version>
<forbiddenapis.version>3.11</forbiddenapis.version>
- <groovy.maven.version>2.1.1</groovy.maven.version>
<maven.antrun.version>3.2.0</maven.antrun.version>
<maven.assembly.version>3.8.0</maven.assembly.version>
<maven.bundle.version>6.2.0</maven.bundle.version>
diff --git a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
index 6ebaecea27..45d62d3be9 100644
--- a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
+++ b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
@@ -111,6 +111,7 @@
<properties>
<!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
<forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+
<opennlp.models.url>https://opennlp.sourceforge.net/models-1.5</opennlp.models.url>
</properties>
<build>
@@ -125,53 +126,52 @@
</inputExcludes>
</configuration>
</plugin>
+ <!-- OpenNLP NER models for NamedEntityParserTest. Best effort: a
failed download
+ or checksum mismatch only warns, and the tests that need the
models skip. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>opennlp-models</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <skip>${skipTests}</skip>
+ <target xmlns:if="ant:if"
xmlns:unless="ant:unless">
+ <property name="cache"
value="${settings.localRepository}/.cache/tika-opennlp-models"/>
+ <mkdir dir="${cache}"/>
+ <macrodef name="model">
+ <attribute name="name"/>
+ <attribute name="sha256"/>
+ <sequential>
+ <local name="present"/>
+ <local name="verified"/>
+ <get
src="${opennlp.models.url}/en-ner-@{name}.bin"
dest="${cache}/en-ner-@{name}.bin"
+ skipexisting="true"
ignoreerrors="true" retries="3"/>
+ <available
file="${cache}/en-ner-@{name}.bin" property="present"/>
+ <checksum if:set="present"
file="${cache}/en-ner-@{name}.bin" algorithm="SHA-256"
+ property="@{sha256}"
verifyproperty="verified"/>
+ <copy if:true="${verified}"
file="${cache}/en-ner-@{name}.bin"
+
tofile="${project.build.testOutputDirectory}/org/apache/tika/parser/ner/opennlp/ner-@{name}.bin"/>
+ <echo unless:true="${verified}"
level="warning"
+ message="OpenNLP model
en-ner-@{name}.bin unavailable or corrupt; NER tests will skip"/>
+ <delete unless:true="${verified}"
file="${cache}/en-ner-@{name}.bin" quiet="true"/>
+ </sequential>
+ </macrodef>
+ <model name="person"
sha256="687a9263d96b37fced707c9f2ac0560f9edaf54658856395555901924f64dbe4"/>
+ <model name="location"
sha256="8fe39e48633f4a86c4132d9c54b396a2d8e0460c1d71e3562dacf976984f447b"/>
+ <model name="organization"
sha256="0136c12afe1ac357142260c39bb879b7c9d121e41024114db5a6455b4fd5ba00"/>
+ <model name="date"
sha256="1207030923852e1c244919d8f15d9e78c217323728fcf909029abd1703967855"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>testSetup</id>
- <activation>
- <!-- auto activate -->
- <file>
-
<missing>src/test/resources/org/apache/tika/parser/ner/opennlp/ner-person.bin</missing>
- </file>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.gmaven</groupId>
- <artifactId>groovy-maven-plugin</artifactId>
- <version>${groovy.maven.version}</version>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-model</artifactId>
- <version>3.9.16</version>
- </dependency>
- <dependency>
- <groupId>org.apache.groovy</groupId>
- <artifactId>groovy-all</artifactId>
- <version>6.0.0</version>
- <type>pom</type>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>testSetup</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>execute</goal>
- </goals>
- <configuration>
-
<source>${basedir}/src/test/resources/org/apache/tika/parser/ner/opennlp/ModelGetter.groovy</source>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
<scm>
<tag>4.1.0-rc1</tag>
</scm>
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/NamedEntityParserTest.java
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/NamedEntityParserTest.java
index c9baf5d368..1ddffb03ea 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/NamedEntityParserTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/NamedEntityParserTest.java
@@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
@@ -41,8 +42,24 @@ public class NamedEntityParserTest extends TikaTest {
public static final String CONFIG_FILE = "tika-config.json";
+ @AfterEach
+ public void clearNerImpl() {
+ System.clearProperty(NamedEntityParser.SYS_PROP_NER_IMPL);
+ }
+
+ // downloaded at build time, best effort; see the antrun plugin in the pom
+ private static void assumeModels() {
+ for (String model : new String[]{OpenNLPNERecogniser.NER_PERSON_MODEL,
+ OpenNLPNERecogniser.NER_LOCATION_MODEL,
OpenNLPNERecogniser.NER_ORGANIZATION_MODEL,
+ OpenNLPNERecogniser.NER_DATE_MODEL}) {
+
assumeTrue(NamedEntityParserTest.class.getClassLoader().getResource(model) !=
null,
+ "missing OpenNLP model " + model);
+ }
+ }
+
@Test
public void testParse() throws Exception {
+ assumeModels();
//test config is added to resources directory
Parser parser = TikaLoader.load(
getConfigPath(NamedEntityParserTest.class,
CONFIG_FILE))
@@ -56,23 +73,23 @@ public class NamedEntityParserTest extends TikaTest {
HashSet<String> set = new HashSet<>(
Arrays.asList(md.getValues(TikaCoreProperties.TIKA_PARSED_BY)));
- assumeTrue(set.contains(NamedEntityParser.class.getName()));
+ assertTrue(set.contains(NamedEntityParser.class.getName()));
set.clear();
set.addAll(Arrays.asList(md.getValues("ner:PERSON")));
- assumeTrue(set.contains("John McKay"));
+ assertTrue(set.contains("John McKay"));
set.clear();
set.addAll(Arrays.asList(md.getValues("ner:LOCATION")));
- assumeTrue(set.contains("Los Angeles"));
+ assertTrue(set.contains("Los Angeles"));
set.clear();
set.addAll(Arrays.asList(md.getValues("ner:ORGANIZATION")));
- assumeTrue(set.contains("University of Southern California"));
+ assertTrue(set.contains("University of Southern California"));
set.clear();
set.addAll(Arrays.asList(md.getValues("ner:DATE")));
- assumeTrue(set.contains("1960 - 1975"));
+ assertTrue(set.contains("1960 - 1975"));
}
@Test
@@ -90,6 +107,7 @@ public class NamedEntityParserTest extends TikaTest {
parser, new Metadata()).metadata;
HashSet<String> keys = new HashSet<>(Arrays.asList(md.names()));
assertTrue(keys.contains("ner:WEEK_DAY"));
- assumeTrue(keys.contains("ner:LOCATION"));
+ assumeModels();
+ assertTrue(keys.contains("ner:LOCATION"));
}
}
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/nltk/NLTKNERecogniserTest.java
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/nltk/NLTKNERecogniserTest.java
index 8858d31a5f..0272f89ae2 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/nltk/NLTKNERecogniserTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/nltk/NLTKNERecogniserTest.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
@@ -34,6 +35,11 @@ import org.apache.tika.parser.ner.NamedEntityParser;
public class NLTKNERecogniserTest extends TikaTest {
+ @AfterEach
+ public void clearNerImpl() {
+ System.clearProperty(NamedEntityParser.SYS_PROP_NER_IMPL);
+ }
+
@Test
public void testGetEntityTypes() throws Exception {
String text = "America is a big country.";
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/regex/RegexNERecogniserTest.java
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/regex/RegexNERecogniserTest.java
index a494ee373b..94e29b4b79 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/regex/RegexNERecogniserTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/java/org/apache/tika/parser/ner/regex/RegexNERecogniserTest.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
@@ -34,6 +35,11 @@ import org.apache.tika.parser.ner.NamedEntityParser;
public class RegexNERecogniserTest extends TikaTest {
+ @AfterEach
+ public void clearNerImpl() {
+ System.clearProperty(NamedEntityParser.SYS_PROP_NER_IMPL);
+ }
+
@Test
public void testGetEntityTypes() throws Exception {
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/ModelGetter.groovy
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/ModelGetter.groovy
deleted file mode 100644
index cf054f2ec1..0000000000
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/ModelGetter.groovy
+++ /dev/null
@@ -1,119 +0,0 @@
-package org.apache.tika.parser.ner.opennlp
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * This file downloads Apache OpenNLP NER models for testing the
NamedEntityParser
- */
-
-import org.apache.maven.settings.Proxy as MvnProxy
-import java.net.Proxy as JDKProxy
-
-/**
- * Copies input stream to output stream, additionally printing the progress.
- * NOTE: this is optimized for large content
- * @param inStr source stream
- * @param outStr target stream
- * @param totalLength the total length of the content (used to calculate
progress)
- * @return
- */
-def copyWithProgress(InputStream inStr, OutputStream outStr, long totalLength){
- int PROGRESS_DELAY = 1000;
- byte[] buffer = new byte[1024 * 4]
- long count = 0
- int len
-
- long tt = System.currentTimeMillis()
- while ((len = inStr.read(buffer)) > 0) {
- outStr.write(buffer, 0, len)
- count += len
- if (System.currentTimeMillis() - tt > PROGRESS_DELAY) {
- println "${count * 100.0/totalLength}% : $count bytes of
$totalLength"
- tt = System.currentTimeMillis()
- }
- }
- println "Copy complete. "
- inStr.close()
- outStr.close()
-}
-
-/**
- * Downloads file
- * @param urlStr url of file
- * @param file path to store file
- * @return
- */
-def downloadFile(String urlStr, File file, Proxy proxy) {
- println "GET : $urlStr -> $file (Using proxy? ${proxy != null})"
- url = new URL(urlStr)
- try {
- urlConn = proxy ? url.openConnection(proxy) : url.openConnection()
- contentLength = urlConn.getContentLengthLong()
-
- file.getParentFile().mkdirs()
- inStream = urlConn.getInputStream()
- outStream = new FileOutputStream(file)
- copyWithProgress(inStream, outStream, contentLength)
- outStream.close()
- inStream.close()
- println "Download Complete.."
- } catch (IOException e) {
- println "Couldn't download $file at the moment. Will skip tests that
require that model/file for now."
- e.printStackTrace()
- }
-}
-
-def proxy = null
-def mvnProxies = settings.getProxies()?.findAll{it.isActive()}
-if (mvnProxies && mvnProxies.size() > 0) {
- mvnProxy = mvnProxies.get(0)
- println "Using the first Proxy setting : ${mvnProxy.username}@
${mvnProxy.host} : ${mvnProxy.port} "
- proxy = new JDKProxy(JDKProxy.Type.HTTP, new
InetSocketAddress(mvnProxy.host, mvnProxy.port))
- Authenticator.setDefault(new Authenticator(){
- @Override
- protected PasswordAuthentication getPasswordAuthentication(){
- return new PasswordAuthentication(mvnProxy.username,
mvnProxy.password?.toCharArray())
- }
- })
- println "Proxy is configured"
-}
-
-def urlPrefix = "http://opennlp.sourceforge.net/models-1.5";
-def prefixPath = "src/test/resources/org/apache/tika/parser/ner/opennlp/"
-
-// detecting proper path for test resources
-if (new File("tika-parsers").exists() && new File("tika-app").exists() ) {
- // running from parent maven project, but resources should go to sub-module
- prefixPath = "tika-parsers/tika-parsers-ml/tika-parser-nlp-module/" +
prefixPath
-}
-
-def modelFiles = //filePath : url
- [(prefixPath + "ner-person.bin"): (urlPrefix + "/en-ner-person.bin"),
- (prefixPath + "ner-location.bin"): (urlPrefix +
"/en-ner-location.bin"),
- (prefixPath + "ner-organization.bin"): (urlPrefix +
"/en-ner-organization.bin"),
- (prefixPath + "en-pos-maxent.bin"): (urlPrefix +
"/en-pos-maxent.bin"),
- (prefixPath + "en-sent.bin"): (urlPrefix + "/en-sent.bin"),
- (prefixPath + "en-token.bin"): (urlPrefix + "/en-token.bin"),
- (prefixPath + "ner-date.bin"): (urlPrefix + "/en-ner-date.bin")]
-
-for (def entry : modelFiles) {
- File file = new File(entry.key)
- if (!file.exists()) {
- downloadFile(entry.value, file, proxy)
- }
-}
\ No newline at end of file
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/get-models.sh
b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/get-models.sh
deleted file mode 100755
index c17899e1ba..0000000000
---
a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/test/resources/org/apache/tika/parser/ner/opennlp/get-models.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "Getting OpenNLP NER models"
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-person.bin"; -O
ner-person.bin
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-location.bin"; -O
ner-location.bin
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-organization.bin"; -O
ner-organization.bin
-
-# Additional 4
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-date.bin"; -O
ner-date.bin
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-money.bin"; -O
ner-money.bin
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-time.bin"; -O
ner-time.bin
-wget "http://opennlp.sourceforge.net/models-1.5/en-ner-percentage.bin"; -O
ner-percentage.bin
\ No newline at end of file
