This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-469
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-469 by this push:
new af73c07 WIP.
af73c07 is described below
commit af73c071a9c84593a28755ea48b7bc86277d5b6f
Author: Sergey Kamov <[email protected]>
AuthorDate: Thu Dec 23 13:16:38 2021 +0300
WIP.
---
.../nlp/token/enricher/impl/NCEnDictionaryImpl.scala | 1 +
.../nlp/token/parser/opennlp/impl/NCOpenNlpImpl.scala | 4 +++-
.../parser/opennlp/NCOpenNlpTokenParserBenchmark.java | 11 +++++++----
.../token/enricher/NCEnDictionaryTokenEnricherSpec.scala | 2 +-
.../token/parser/opennlp/NCOpenNlpTokenParserSpec.scala | 14 +++++---------
.../apache/nlpcraft/internal/nlp/util/NCTestUtils.scala | 8 ++++----
6 files changed, 21 insertions(+), 19 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/enricher/impl/NCEnDictionaryImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/enricher/impl/NCEnDictionaryImpl.scala
index bfb8e26..405cff3 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/enricher/impl/NCEnDictionaryImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/enricher/impl/NCEnDictionaryImpl.scala
@@ -26,6 +26,7 @@ class NCEnDictionaryImpl extends NCLifecycle:
@volatile private var dict: Set[String] = _
override def start(): Unit = dict =
NCUtils.readResource("moby/354984si.ngl", "iso-8859-1").toSet
+ override def stop(): Unit = dict = null
/**
*
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCOpenNlpImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCOpenNlpImpl.scala
index 6b25b42..dfc3c69 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCOpenNlpImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCOpenNlpImpl.scala
@@ -83,6 +83,8 @@ class NCOpenNlpImpl(
() => sw = new NCEnStopWordsFinder(stem(addStopWords),
stem(exclStopWords))
)(ExecutionContext.Implicits.global)
+ override def stop(): Unit = sw = null; lemmatizer = null; tagger = null;
lemmatizer = null
+
/**
*
* @param addStopWords
@@ -120,7 +122,7 @@ class NCOpenNlpImpl(
* @return
*/
override def parse(req: NCRequest): JList[NCToken] =
- // TODO: check started? clear on stop?
+ // TODO: check started?
if (tokenizer == null)
throw new IllegalStateException(s"${this.getClass.getName} is not
started.")
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCOpenNlpTokenParserBenchmark.java
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCOpenNlpTokenParserBenchmark.java
index f82a26e..788ef3b 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCOpenNlpTokenParserBenchmark.java
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCOpenNlpTokenParserBenchmark.java
@@ -23,12 +23,15 @@ import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.infra.Blackhole;
+/**
+ *
+ */
public class NCOpenNlpTokenParserBenchmark extends NCBenchmarkAdapter {
- private NCOpenNlpTokenParser prepared;
+ private NCOpenNlpTokenParser parser;
@Setup
public void setUp() {
- prepared = prepareParser();
+ parser = prepareParser();
}
@Benchmark
@@ -37,8 +40,8 @@ public class NCOpenNlpTokenParserBenchmark extends
NCBenchmarkAdapter {
}
@Benchmark
- public void testParse(Blackhole bh,
NCBenchmarkAdapter.NCBenchmarkAdapterState state) {
- bh.consume(prepared.parse(state.request));
+ public void testParse(Blackhole bh, NCBenchmarkAdapterState state) {
+ bh.consume(parser.parse(state.request));
}
/**
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/enricher/NCEnDictionaryTokenEnricherSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/enricher/NCEnDictionaryTokenEnricherSpec.scala
index f8ee857..c48751d 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/enricher/NCEnDictionaryTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/enricher/NCEnDictionaryTokenEnricherSpec.scala
@@ -45,4 +45,4 @@ class NCEnDictionaryTokenEnricherSpec:
enricher.enrich(null, null, toks.asJava)
require(toks.head.get[Boolean]("english"))
- require(!toks.last.get[Boolean]("english"))
+ require(!toks.last.get[Boolean]("english"))
\ No newline at end of file
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCOpenNlpTokenParserSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCOpenNlpTokenParserSpec.scala
index e93192a..e507666 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCOpenNlpTokenParserSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCOpenNlpTokenParserSpec.scala
@@ -43,18 +43,14 @@ class NCOpenNlpTokenParserSpec:
)
)
- private def test(txt: String, validate: Seq[NCToken] => _) =
- val toks = parser.parse(nlp.util.NCTestRequest(txt))
+ private def test(txt: String, validate: Seq[NCToken] => _): Unit =
+ val toks = parser.parse(nlp.util.NCTestRequest(txt)).asScala.toSeq
- assert(toks != null)
+ assert(toks.nonEmpty)
- val toksScala = toks.asScala.toSeq
+ NCTestUtils.printTokens(txt, toks)
- assert(toksScala.nonEmpty)
-
- NCTestUtils.printTokens(txt, toksScala)
-
- validate(toksScala)
+ validate(toks)
@Test
def test(): Unit =
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
index 227ce7b..b810b30 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
@@ -59,12 +59,12 @@ object NCTestUtils:
def makeAndStart[T <: NCLifecycle](make: => T): T =
def now = System.currentTimeMillis()
- val t1 = now
+ val start = now
val t = make
- val t2 = now
-
+ val started = now
+
t.start()
- println(s"Component: '${t.getClass.getSimpleName}' created time=${t2 -
t1} ms and started=${now - t2} ms.")
+ println(s"'${t.getClass.getSimpleName}' created with time=${started -
start} ms and started=${now - started} ms.")
t
