This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-472
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-472 by this push:
new 1648dfe WIP.
1648dfe is described below
commit 1648dfe3e892ab11ac91eeb7913697bd7e8e563e
Author: Sergey Kamov <[email protected]>
AuthorDate: Wed Dec 29 21:55:52 2021 +0300
WIP.
---
.../apache/nlpcraft/nlp/tokenizer/NCOpenNlpTokenizer.java | 3 +++
.../nlp/tokenizer/impl/NCOpenNlpTokenizerImpl.scala | 13 ++++++-------
.../entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala | 2 +-
.../entity/parser/semantic/NCSemanticEntityParserSpec.scala | 2 +-
.../token/enricher/en/NCEnBracketsTokenEnricherSpec.scala | 2 +-
.../token/enricher/en/NCEnDictionaryTokenEnricherSpec.scala | 2 +-
.../nlp/token/enricher/en/NCEnQuotesTokenEnricherSpec.scala | 2 +-
.../parser/opennlp/en/NCEnOpenNlpTokenParserSpec.scala | 2 +-
.../util/{NCTestDefaultConfig.scala => NCTestConfig.scala} | 2 +-
9 files changed, 16 insertions(+), 14 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/NCOpenNlpTokenizer.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/NCOpenNlpTokenizer.java
index 03e8c9a..3e1aeb9 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/NCOpenNlpTokenizer.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/NCOpenNlpTokenizer.java
@@ -26,6 +26,9 @@ import java.io.File;
import java.util.List;
import java.util.Objects;
+/**
+ *
+ */
public class NCOpenNlpTokenizer implements NCTokenizer {
private final NCOpenNlpTokenizerImpl impl;
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/impl/NCOpenNlpTokenizerImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/impl/NCOpenNlpTokenizerImpl.scala
index 18d54b4..05729a5 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/impl/NCOpenNlpTokenizerImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/tokenizer/impl/NCOpenNlpTokenizerImpl.scala
@@ -34,11 +34,10 @@ class NCOpenNlpTokenizerImpl(is: InputStream) extends
NCTokenizer:
override def start(cfg: NCModelConfig): Unit = tokenizer = new
TokenizerME(new TokenizerModel(is))
override def stop(): Unit = tokenizer = null
override def tokenize(cfg: NCModelConfig, txt: String): util.List[NCWord] =
- tokenizer.tokenizePos(txt).
- map(span =>
- new NCWord:
- override def getText: String =
span.getCoveredText(txt).toString
- override def getStartCharIndex: Int = span.getStart
- override def getEndCharIndex: Int = span.getEnd
- override def getLength: Int = span.length()
+ this.synchronized { tokenizer.tokenizePos(txt) }.
+ map(span => new NCWord :
+ override def getText: String =
span.getCoveredText(txt).toString
+ override def getStartCharIndex: Int = span.getStart
+ override def getEndCharIndex: Int = span.getEnd
+ override def getLength: Int = span.length()
).toSeq.asJava
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
index f705e44..a9cfb14 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
@@ -22,7 +22,7 @@ import org.apache.nlpcraft.internal.util.NCUtils
import org.apache.nlpcraft.nlp.entity.parser.opennlp.NCOpenNlpEntityParser
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.junit.jupiter.api.*
import java.util
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
index 309ec6c..041d900 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
@@ -22,7 +22,7 @@ import org.apache.nlpcraft.internal.util.NCUtils
import org.apache.nlpcraft.nlp.entity.parser.opennlp.NCOpenNlpEntityParser
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.junit.jupiter.api.*
import java.util
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnBracketsTokenEnricherSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnBracketsTokenEnricherSpec.scala
index 6fd2210..a4d07b1 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnBracketsTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnBracketsTokenEnricherSpec.scala
@@ -21,7 +21,7 @@ import org.apache.nlpcraft.*
import org.apache.nlpcraft.nlp.token.enricher.en.NCEnBracketsTokenEnricher
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.junit.jupiter.api.*
import scala.jdk.CollectionConverters.*
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnDictionaryTokenEnricherSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnDictionaryTokenEnricherSpec.scala
index 1bceab7..bd03637 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnDictionaryTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnDictionaryTokenEnricherSpec.scala
@@ -20,7 +20,7 @@ package org.apache.nlpcraft.nlp.token.enricher.en
import org.apache.nlpcraft.nlp.token.enricher.en.NCEnDictionaryTokenEnricher
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.junit.jupiter.api.*
import scala.jdk.CollectionConverters.SeqHasAsJava
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnQuotesTokenEnricherSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnQuotesTokenEnricherSpec.scala
index e3c6bd1..c8ca5cb 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnQuotesTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/enricher/en/NCEnQuotesTokenEnricherSpec.scala
@@ -20,7 +20,7 @@ package org.apache.nlpcraft.nlp.token.enricher.en
import org.apache.nlpcraft.NCToken
import org.apache.nlpcraft.nlp.token.enricher.en.NCEnQuotesTokenEnricher
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.apache.nlpcraft.nlp.util.*
import org.junit.jupiter.api.*
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/parser/opennlp/en/NCEnOpenNlpTokenParserSpec.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/parser/opennlp/en/NCEnOpenNlpTokenParserSpec.scala
index e41f2ac..2311889 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/parser/opennlp/en/NCEnOpenNlpTokenParserSpec.scala
+++
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/token/parser/opennlp/en/NCEnOpenNlpTokenParserSpec.scala
@@ -21,7 +21,7 @@ import org.apache.nlpcraft.*
import org.apache.nlpcraft.internal.ascii.NCAsciiTable
import org.apache.nlpcraft.nlp.token.parser.opennlp.en.NCEnOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
-import org.apache.nlpcraft.nlp.util.NCTestDefaultConfig.*
+import org.apache.nlpcraft.nlp.util.NCTestConfig.*
import org.junit.jupiter.api.*
import java.util
diff --git
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestDefaultConfig.scala
b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestConfig.scala
similarity index 98%
rename from
nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestDefaultConfig.scala
rename to nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestConfig.scala
index e0016ab..46242ef 100644
---
a/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestDefaultConfig.scala
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/nlp/util/NCTestConfig.scala
@@ -24,7 +24,7 @@ import org.apache.nlpcraft.*
import java.util
import java.util.Optional
-object NCTestDefaultConfig:
+object NCTestConfig:
final val EN_TOKENIZER = new NCOpenNlpTokenizer(
"opennlp/en-token.bin"
)
