This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/master by this push:
new 0221abf Tests infrastructure improved.
0221abf is described below
commit 0221abf8cfa63fdf9c633610797124fd42c3363d
Author: Sergey Kamov <[email protected]>
AuthorDate: Thu Mar 17 11:56:55 2022 +0300
Tests infrastructure improved.
---
.../apache/nlpcraft/NCModelPipelineBuilder.java | 2 +-
.../internal/conversation/NCConversationData.scala | 2 +-
.../impl/NCEnStopWordsTokenEnricherImpl.scala | 3 +-
.../conversation/NCConversationManagerSpec.scala | 2 +-
.../nlpcraft/internal/util/NCResourceSpec.scala | 8 ++++--
.../semantic/NCSemanticEntityParserSpec.scala | 2 +-
.../semantic/NCSemanticEntityParserYamlSpec.scala | 2 +-
.../enricher/NCBracketsTokenEnricherSpec.scala | 4 +--
.../token/enricher/NCStopWordsEnricherSpec.scala | 2 +-
.../enricher/NCSwearWordsTokenEnricherSpec.scala | 4 +--
.../apache/nlpcraft/nlp/util/NCTestPipeline.scala | 18 ++----------
.../org/apache/nlpcraft/nlp/util/NCTestUtils.scala | 32 +++++++++-------------
12 files changed, 32 insertions(+), 49 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelPipelineBuilder.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelPipelineBuilder.java
index 631015b..991ddb9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelPipelineBuilder.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelPipelineBuilder.java
@@ -232,7 +232,7 @@ public class NCModelPipelineBuilder {
));
tokEnrichers.add(new NCEnStopWordsTokenEnricher());
- // TODO: Do we need following components here?
+ // TODO: Do we need the following components here?
tokEnrichers.add(new
NСEnSwearWordsTokenEnricher(NCResourceReader.getPath("badfilter/swear_words.txt")));
tokEnrichers.add(new NCEnQuotesTokenEnricher());
tokEnrichers.add(new NCEnDictionaryTokenEnricher());
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
index 88cc991..472dc6a 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
@@ -202,7 +202,7 @@ case class NCConversationData(
}
/**
- * TODO: thread safe?
+ *
*/
val getUserData: NCPropertyMap = data
}
\ No newline at end of file
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/impl/NCEnStopWordsTokenEnricherImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/impl/NCEnStopWordsTokenEnricherImpl.scala
index 31a3a3f..efcf04f 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/impl/NCEnStopWordsTokenEnricherImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/impl/NCEnStopWordsTokenEnricherImpl.scala
@@ -113,8 +113,7 @@ object NCEnStopWordsTokenEnricherImpl:
* @param tokens Tokens.
* @param maxLen Maximum number of tokens in the sequence.
*/
- // TODO: private[impl]
- def tokenMixWithStopWords(tokens: Seq[NCToken], maxLen: Int =
Integer.MAX_VALUE): Seq[Seq[NCToken]] =
+ private[impl] def tokenMixWithStopWords(tokens: Seq[NCToken], maxLen: Int
= Integer.MAX_VALUE): Seq[Seq[NCToken]] =
/**
* Gets all combinations for sequence of mandatory tokens with
stop-words and without.
*
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
index cf23b87..113b538 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
@@ -65,7 +65,7 @@ class NCConversationManagerSpec:
val t = NCTestToken()
val reqId = "req1"
- // TODO: Drop method and use saved conversation instead - error is
thrown.
+ // TODO: important (error in code) - drop method and use saved
conversation instead - error is thrown.
def getConversation: NCConversationData = mgr.getConversation("user1")
def checkSize(size: Int): Unit =
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/util/NCResourceSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/util/NCResourceSpec.scala
index 9d50975..2ffae4b 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/util/NCResourceSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/util/NCResourceSpec.scala
@@ -25,15 +25,17 @@ import org.junit.jupiter.api.Test
class NCResourceSpec:
@Test
def test(): Unit =
+ val path = "opennlp/en-token.bin"
+
// Get and delete.
- var f = NCResourceReader.get("opennlp/en-lemmatizer.dict")
+ var f = NCResourceReader.get(path)
require(f.delete())
println(s"Deleted: ${f.getAbsolutePath}")
// Download.
- f = NCResourceReader.get("opennlp/en-lemmatizer.dict")
+ f = NCResourceReader.get(path)
// From cache.
- f = NCResourceReader.get("opennlp/en-lemmatizer.dict")
+ f = NCResourceReader.get(path)
// By absolute path.
f = NCResourceReader.get(f.getAbsolutePath)
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
index d298673..71935ea 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
@@ -35,7 +35,7 @@ import scala.jdk.OptionConverters.RichOptional
*/
class NCSemanticEntityParserSpec:
import NCSemanticTestElement as E
- private val semParser =
+ private val semParser: NCSemanticEntityParser =
NCTestUtils.mkENSemanticParser(
Seq(
// Standard.
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
index 3a4702c..bf2534b 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
@@ -32,7 +32,7 @@ import scala.jdk.OptionConverters.RichOptional
*
*/
class NCSemanticEntityParserYamlSpec:
- private val semParser =
NCTestUtils.mkENSemanticParser("models/lightswitch_model.yaml")
+ private val semParser: NCSemanticEntityParser =
NCTestUtils.mkENSemanticParser("models/lightswitch_model.yaml")
/**
*
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCBracketsTokenEnricherSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCBracketsTokenEnricherSpec.scala
index 9b07783..edae641 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCBracketsTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCBracketsTokenEnricherSpec.scala
@@ -28,7 +28,7 @@ import scala.jdk.CollectionConverters.*
*
*/
class NCBracketsTokenEnricherSpec:
- private val enricher = new NCEnBracketsTokenEnricher()
+ private val bracketsEnricher = new NCEnBracketsTokenEnricher()
/**
*
@@ -37,7 +37,7 @@ class NCBracketsTokenEnricherSpec:
*/
private def check(txt: String, brackets: Set[Integer]): Unit =
val toks = EN_TOK_PARSER.tokenize(txt)
- enricher.enrich(NCTestRequest(txt), CFG, toks)
+ bracketsEnricher.enrich(NCTestRequest(txt), CFG, toks)
val seq = toks.asScala.toSeq
NCTestUtils.printTokens(seq)
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCStopWordsEnricherSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCStopWordsEnricherSpec.scala
index 8bdd11a..bfb5e48 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCStopWordsEnricherSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCStopWordsEnricherSpec.scala
@@ -54,7 +54,7 @@ class NCStopWordsEnricherSpec:
@Test
def test(): Unit =
test(
- new NCEnStopWordsTokenEnricher(),
+ EN_TOK_STOP_ENRICHER,
"the test",
true,
false
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCSwearWordsTokenEnricherSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCSwearWordsTokenEnricherSpec.scala
index 35ada43..6afebe6 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCSwearWordsTokenEnricherSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/NCSwearWordsTokenEnricherSpec.scala
@@ -29,7 +29,7 @@ import scala.jdk.CollectionConverters.*
*
*/
class NCSwearWordsTokenEnricherSpec:
- private val enricher = new
NСEnSwearWordsTokenEnricher(NCResourceReader.getPath("badfilter/swear_words.txt"))
+ private val swEnricher = new
NСEnSwearWordsTokenEnricher(NCResourceReader.getPath("badfilter/swear_words.txt"))
@Test
def test(): Unit =
@@ -38,7 +38,7 @@ class NCSwearWordsTokenEnricherSpec:
require(toks.head.getOpt[Boolean]("swear:en").isEmpty)
require(toks.last.getOpt[Boolean]("swear:en").isEmpty)
- enricher.enrich(null, null, toks.asJava)
+ swEnricher.enrich(null, null, toks.asJava)
NCTestUtils.printTokens(toks)
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestPipeline.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestPipeline.scala
index 8c69f06..d022757 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestPipeline.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestPipeline.scala
@@ -27,10 +27,10 @@ import java.util.{Optional, ArrayList as JList}
*
* @param tokParser
*/
-case class NCTestPipeline(tokParser: NCTokenParser) extends
NCPropertyMapAdapter with NCModelPipeline with Cloneable:
+case class NCTestPipeline(tokParser: NCTokenParser) extends
NCPropertyMapAdapter with NCModelPipeline:
require(tokParser != null)
- private var variantFilter: Optional[NCVariantFilter] = Optional.empty()
+ var variantFilter: Optional[NCVariantFilter] = Optional.empty()
override val getTokenParser: NCTokenParser = tokParser
override val getTokenEnrichers = new JList[NCTokenEnricher]()
@@ -38,16 +38,4 @@ case class NCTestPipeline(tokParser: NCTokenParser) extends
NCPropertyMapAdapter
override val getEntityParsers = new JList[NCEntityParser]()
override val getTokenValidators = new JList[NCTokenValidator]()
override val getEntityValidators = new JList[NCEntityValidator]()
- override def getVariantFilter: Optional[NCVariantFilter] = variantFilter
-
- override def clone(): NCTestPipeline =
- val copy = NCTestPipeline(this.tokParser)
-
- copy.getTokenEnrichers.addAll(this.getTokenEnrichers)
- copy.getEntityEnrichers.addAll(this.getEntityEnrichers)
- copy.getEntityParsers.addAll(this.getEntityParsers)
- copy.getTokenValidators.addAll(this.getTokenValidators)
- copy.getEntityValidators.addAll(this.getEntityValidators)
- copy.variantFilter = this.variantFilter
-
- copy
\ No newline at end of file
+ override def getVariantFilter: Optional[NCVariantFilter] = variantFilter
\ No newline at end of file
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestUtils.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestUtils.scala
index f8ada63..b73af1d 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestUtils.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestUtils.scala
@@ -28,6 +28,7 @@ import java.util.{Map, List as JList}
import scala.jdk.CollectionConverters.*
import scala.jdk.OptionConverters.RichOptional
import scala.util.Using
+import opennlp.tools.stemmer.PorterStemmer
/**
*
@@ -123,21 +124,22 @@ object NCTestUtils:
/**
*
+ * @return
+ */
+ private def mkSemanticStemmer: NCSemanticStemmer =
+ new NCSemanticStemmer():
+ private val ps = new PorterStemmer
+ override def stem(txt: String): String = ps.synchronized {
ps.stem(txt.toLowerCase) }
+
+
+ /**
+ *
* @param elms
* @param macros
* @return
*/
def mkENSemanticParser(elms: util.List[NCSemanticElement], macros:
util.Map[String, String] = null): NCSemanticEntityParser =
- val s = new opennlp.tools.stemmer.PorterStemmer
-
- new NCSemanticEntityParser(
- new NCSemanticStemmer():
- override def stem(txt: String): String =
s.stem(txt.toLowerCase)
- ,
- EN_TOK_PARSER,
- macros,
- elms
- )
+ new NCSemanticEntityParser(mkSemanticStemmer, EN_TOK_PARSER, macros,
elms)
/**
*
@@ -145,12 +147,4 @@ object NCTestUtils:
* @return
*/
def mkENSemanticParser(src: String): NCSemanticEntityParser =
- val s = new opennlp.tools.stemmer.PorterStemmer
-
- new NCSemanticEntityParser(
- new NCSemanticStemmer():
- override def stem(txt: String): String =
s.stem(txt.toLowerCase)
- ,
- EN_TOK_PARSER,
- src
- )
+ new NCSemanticEntityParser(mkSemanticStemmer, EN_TOK_PARSER, src)
