This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-472
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-472 by this push:
new 755c465 CR WIP
755c465 is described below
commit 755c465529aca88d6c0ec58d6da5b3486df3fa0a
Author: Aaron Radzinski <[email protected]>
AuthorDate: Sun Jan 16 13:26:43 2022 -0800
CR WIP
---
.../main/scala/org/apache/nlpcraft/NCContext.java | 1 -
.../scala/org/apache/nlpcraft/NCConversation.java | 8 ++++-
.../main/scala/org/apache/nlpcraft/NCEntity.java | 6 ++++
.../nlp/entity/parser/nlp/NCNlpEntityParser.java | 9 ++----
...NCNlpEntityParserImpl.scala => NCNlpImpl.scala} | 7 ++--
.../parser/opennlp/NCOpenNlpEntityParser.java | 17 ++++------
...pEntityParserImpl.scala => NCOpenNlpImpl.scala} | 37 +++++++++++-----------
.../internal/NCModelPipelineProcessorSpec.scala | 27 +++++++++-------
.../entity/parser/nlp/NCNlpEntityParserSpec.scala | 3 ++
.../parser/opennlp/NCOpenNlpEntityParserSpec.scala | 8 +++++
.../semantic/NCSemanticEntityParserJsonSpec.scala | 9 ++++++
.../semantic/NCSemanticEntityParserSpec.scala | 14 +++++---
.../semantic/NCSemanticEntityParserYamlSpec.scala | 8 +++++
.../nlpcraft/nlp/util/opennlp/NCTestConfig.scala | 1 +
14 files changed, 97 insertions(+), 58 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCContext.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCContext.java
index 36a4a51..1db8097 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCContext.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCContext.java
@@ -27,7 +27,6 @@ public interface NCContext {
* Tests if given entity is part of the query this context is associated
with.
*
* @param ent Entity to check.
- * @return {@code true} if given entity is associated with this context,
{@code false} otherwise.
*/
boolean isOwnerOf(NCEntity ent);
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
index 6cfc375..7856601 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
@@ -25,9 +25,15 @@ import java.util.function.Predicate;
*/
public interface NCConversation {
/**
+ *
+ * @return
+ */
+ NCPropertyMap getSession();
+
+ /**
*
*/
- List<NCEntity> getEntities();
+ List<NCEntity> getStm();
/**
* Gets the chronologically ordered list of previously matched intents
sorted from oldest to newest
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCEntity.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCEntity.java
index ca746bb..22fc53d 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCEntity.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCEntity.java
@@ -37,6 +37,12 @@ public interface NCEntity extends NCPropertyMap {
String getRequestId();
/**
+ *
+ * @return
+ */
+ default String getGroup() { return getId(); }
+
+ /**
*
* @return
*/
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParser.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParser.java
index 53165e7..2e57674 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParser.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParser.java
@@ -22,22 +22,17 @@ import org.apache.nlpcraft.NCEntityParser;
import org.apache.nlpcraft.NCModelConfig;
import org.apache.nlpcraft.NCRequest;
import org.apache.nlpcraft.NCToken;
-import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpEntityParserImpl;
+import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpImpl;
import java.util.List;
/**
* TODO: add javadoc based on comments below.
*
- * Umbrella for NLP tokens.
- *
- * Each entity with ID 'nlp:token' contans one token and have following
properties copied from its token:
- * nlp:token:lemma, nlp:token:pos, nlp:token:text, nlp:token:index
- *
* Component is language independent.
*/
public class NCNlpEntityParser implements NCEntityParser {
- private final NCNlpEntityParserImpl impl = new NCNlpEntityParserImpl();
+ private final NCNlpImpl impl = new NCNlpImpl();
@Override
public List<NCEntity> parse(NCRequest req, NCModelConfig cfg,
List<NCToken> toks) {
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpEntityParserImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpImpl.scala
similarity index 90%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpEntityParserImpl.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpImpl.scala
index c8c4367..c60aa9f 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpEntityParserImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/impl/NCNlpImpl.scala
@@ -26,19 +26,18 @@ import java.util.stream.Collectors
/**
*
*/
-object NCNlpEntityParserImpl:
+object NCNlpImpl:
private def id = "nlp:token"
-import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpEntityParserImpl.*
+import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpImpl.*
/**
*
*/
-class NCNlpEntityParserImpl extends NCEntityParser:
+class NCNlpImpl extends NCEntityParser:
override def parse(req: NCRequest, cfg: NCModelConfig, toks:
JList[NCToken]): JList[NCEntity] =
toks.stream().map(t =>
new NCPropertyMapAdapter with NCEntity:
- // TODO: add stem?
put(s"$id:lemma", t.getLemma)
put(s"$id:pos", t.getPos)
put(s"$id:text", t.getText)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
index 465c941..d82cb42 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
@@ -17,12 +17,8 @@
package org.apache.nlpcraft.nlp.entity.parser.opennlp;
-import org.apache.nlpcraft.NCEntity;
-import org.apache.nlpcraft.NCEntityParser;
-import org.apache.nlpcraft.NCModelConfig;
-import org.apache.nlpcraft.NCRequest;
-import org.apache.nlpcraft.NCToken;
-import
org.apache.nlpcraft.nlp.entity.parser.opennlp.impl.NCOpenNlpEntityParserImpl;
+import org.apache.nlpcraft.*;
+import org.apache.nlpcraft.nlp.entity.parser.opennlp.impl.NCOpenNlpImpl;
import java.util.List;
import java.util.Objects;
@@ -41,26 +37,25 @@ import java.util.Objects;
* <p>
*/
public class NCOpenNlpEntityParser implements NCEntityParser {
- private final NCOpenNlpEntityParserImpl impl;
+ private final NCOpenNlpImpl impl;
/**
* @param mdlSrc
*/
public NCOpenNlpEntityParser(String mdlSrc) {
- // TODO: error texts.
Objects.requireNonNull(mdlSrc, "Model source cannot be null.");
- this.impl = new
NCOpenNlpEntityParserImpl(java.util.Collections.singletonList(mdlSrc));
+ this.impl = new
NCOpenNlpImpl(java.util.Collections.singletonList(mdlSrc));
}
/**
* @param mdlSrcs
*/
public NCOpenNlpEntityParser(List<String> mdlSrcs) {
- // TODO: error texts.
Objects.requireNonNull(mdlSrcs, "Model sources cannot be null.");
+ if (mdlSrcs.size() == 0) throw new NCException("Model sources cannot
be empty.");
- this.impl = new NCOpenNlpEntityParserImpl(mdlSrcs);
+ this.impl = new NCOpenNlpImpl(mdlSrcs);
}
@Override
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpImpl.scala
similarity index 81%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpImpl.scala
index eef48cc..fd26880 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/impl/NCOpenNlpImpl.scala
@@ -34,38 +34,40 @@ import scala.util.Using
/**
*
- * @param resources
+ * @param srcs
*/
-class NCOpenNlpEntityParserImpl(resources: JList[String]) extends
NCEntityParser with LazyLogging :
- require(resources != null)
+class NCOpenNlpImpl(srcs: JList[String]) extends NCEntityParser with
LazyLogging :
+ require(srcs != null)
private var finders: Seq[NameFinderME] = _
+ private case class Holder(start: Int, end: Int, name: String, probability:
Double)
init()
- private case class Holder(start: Int, end: Int, name: String, probability:
Double)
-
+ /**
+ *
+ */
private def init(): Unit =
val finders = mutable.ArrayBuffer.empty[NameFinderME]
-
NCUtils.execPar(
- resources.asScala.toSeq.map(res => () => {
+ srcs.asScala.toSeq.map(res => () => {
val f = new NameFinderME(new
TokenNameFinderModel(NCUtils.getStream(res)))
-
logger.trace(s"Loaded resource: $res")
-
finders.synchronized { finders += f }
})*)(ExecutionContext.Implicits.global)
this.finders = finders.toSeq
- private def find(finder: NameFinderME, words: Array[String]):
Array[Holder] =
- finder.synchronized {
- try
- finder.find(words).map(p => Holder(p.getStart, p.getEnd - 1,
p.getType, p.getProb))
- finally
- finder.clearAdaptiveData()
- }
+ /**
+ *
+ * @param finder
+ * @param words
+ * @return
+ */
+ private def find(finder: NameFinderME, words: Array[String]):
Array[Holder] = finder.synchronized {
+ try finder.find(words).map(p => Holder(p.getStart, p.getEnd - 1,
p.getType, p.getProb))
+ finally finder.clearAdaptiveData()
+ }
override def parse(req: NCRequest, cfg: NCModelConfig, toksList:
JList[NCToken]): JList[NCEntity] =
val toks = toksList.asScala
@@ -84,8 +86,7 @@ class NCOpenNlpEntityParserImpl(resources: JList[String])
extends NCEntityParser
new NCPropertyMapAdapter with NCEntity:
put(s"opennlp:${h.name}:probability", h.probability)
- override val getTokens: JList[NCToken] =
- toks.flatMap(t => Option.when(t.getIndex >= i1 &&
t.getIndex <= i2)(t)).asJava
+ override val getTokens: JList[NCToken] = toks.flatMap(t =>
Option.when(t.getIndex >= i1 && t.getIndex <= i2)(t)).asJava
override val getRequestId: String = req.getRequestId
override val getId: String = s"opennlp:${h.name}"
)
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/NCModelPipelineProcessorSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/NCModelPipelineProcessorSpec.scala
index 6eec102..24db423 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/NCModelPipelineProcessorSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/NCModelPipelineProcessorSpec.scala
@@ -19,7 +19,7 @@ package org.apache.nlpcraft.internal
import org.apache.nlpcraft.*
import org.apache.nlpcraft.internal.impl.NCModelPipelineProcessor
-import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpEntityParserImpl
+import org.apache.nlpcraft.nlp.entity.parser.nlp.impl.NCNlpImpl
import org.apache.nlpcraft.nlp.entity.parser.semantic.*
import org.apache.nlpcraft.nlp.entity.parser.semantic.impl.en.NCEnPorterStemmer
import org.apache.nlpcraft.nlp.util.*
@@ -37,6 +37,9 @@ import scala.jdk.CollectionConverters.*
*
*/
class NCModelPipelineProcessorSpec:
+ /**
+ *
+ */
@Test
def test(): Unit =
def test(txt: String, variantCnt: Int, elements: NCSemanticElement*):
Unit =
@@ -46,9 +49,7 @@ class NCModelPipelineProcessorSpec:
pipeline.getEntityParsers.clear()
pipeline.getEntityParsers.add(parser)
- val res =
- new NCModelPipelineProcessor(new NCModelAdapter(CFG,
pipeline)).
- prepVariants(txt, null, "userId")
+ val res = new NCModelPipelineProcessor(new NCModelAdapter(CFG,
pipeline)).prepVariants(txt, null, "userId")
println(s"Variants count: ${res.vars.size}")
for ((v, idx) <- res.vars.zipWithIndex)
@@ -63,10 +64,10 @@ class NCModelPipelineProcessorSpec:
/**
*
* @param delayMs
- * @param itersCnt
+ * @param iterCnt
* @return
*/
- private def mkSlowPipelineProcessor(delayMs: Long, itersCnt: Int):
NCModelPipelineProcessor =
+ private def mkSlowPipelineProcessor(delayMs: Long, iterCnt: Int):
NCModelPipelineProcessor =
val pipeline = EN_PIPELINE.clone()
pipeline.getEntityParsers.clear()
@@ -78,10 +79,13 @@ class NCModelPipelineProcessorSpec:
Thread.sleep(delayMs)
java.util.Collections.emptyList()
- (0 until itersCnt).foreach(i =>
pipeline.getEntityParsers.add(mkSlowParser(i)))
+ (0 until iterCnt).foreach(i =>
pipeline.getEntityParsers.add(mkSlowParser(i)))
NCModelPipelineProcessor(new NCModelAdapter(CFG, pipeline))
+ /**
+ *
+ */
@Test
def testCancel(): Unit =
val fut = mkSlowPipelineProcessor(1, 10000).ask("any", null, "userId")
@@ -92,13 +96,14 @@ class NCModelPipelineProcessorSpec:
Assertions.assertThrows(classOf[CancellationException], () => fut.get)
+ /**
+ *
+ */
@Test
def testTimeout(): Unit =
val fut = mkSlowPipelineProcessor(1, 10000).ask("any", null, "userId")
Thread.sleep(20)
- try
- Assertions.assertThrows(classOf[TimeoutException], () =>
fut.get(1, TimeUnit.MILLISECONDS))
- finally
- fut.cancel(true)
+ try Assertions.assertThrows(classOf[TimeoutException], () =>
fut.get(1, TimeUnit.MILLISECONDS))
+ finally fut.cancel(true)
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParserSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParserSpec.scala
index eaacd70..1d648f8 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParserSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/nlp/NCNlpEntityParserSpec.scala
@@ -36,6 +36,9 @@ import scala.jdk.OptionConverters.RichOptional
class NCNlpEntityParserSpec:
private val parser = new NCNlpEntityParser()
+ /**
+ *
+ */
@Test
def test(): Unit =
val req = NCTestRequest("I had the lunch")
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
index 26ca973..da30d94 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
@@ -45,6 +45,11 @@ class NCOpenNlpEntityParserSpec:
).asJava
)
+ /**
+ *
+ * @param txt
+ * @param expected
+ */
private def check(txt: String, expected: String): Unit =
val req = NCTestRequest(txt)
val toks = EN_PIPELINE.getTokenParser.tokenize(txt)
@@ -55,6 +60,9 @@ class NCOpenNlpEntityParserSpec:
require(ents.sizeIs == 1)
require(ents.exists(_.getOpt(s"opennlp:$expected:probability").isPresent))
+ /**
+ *
+ */
@Test
def test(): Unit =
check("today", "date")
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserJsonSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserJsonSpec.scala
index 228ee2e..a9bdde7 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserJsonSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserJsonSpec.scala
@@ -42,6 +42,12 @@ class NCSemanticEntityParserJsonSpec:
"models/alarm_model.json"
)
+ /**
+ *
+ * @param txt
+ * @param id
+ * @param elemData
+ */
private def check(txt: String, id: String, elemData: Option[Map[String,
Any]] = None): Unit =
val req = NCTestRequest(txt)
val ents = parser.parse(
@@ -59,6 +65,9 @@ class NCSemanticEntityParserJsonSpec:
case Some(m) => m.foreach { (k, v) =>
require(tok.get[Any](s"$id:$k") == v) }
case None => // No-op.
+ /**
+ *
+ */
@Test
def test(): Unit =
check(
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
index af5bf66..4b8288a 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserSpec.scala
@@ -53,9 +53,12 @@ case class NCSemanticTestElement(
override def getSynonyms: JSet[String] = synonyms.asJava
override def getProperties: JMap[String, Object] = props.asJava
-object NCSemanticTestElement {
+/**
+ *
+ */
+object NCSemanticTestElement:
def apply(id: String, synonyms: String*) = new NCSemanticTestElement(id,
synonyms = synonyms.toSet)
-}
+
/**
*
*/
@@ -92,9 +95,7 @@ class NCSemanticEntityParserSpec:
* @param value
* @param elemData
*/
- private def check(
- txt: String, id: String, value: Option[String] = None, elemData:
Option[Map[String, Any]] = None
- ): Unit =
+ private def check(txt: String, id: String, value: Option[String] = None,
elemData: Option[Map[String, Any]] = None): Unit =
val req = NCTestRequest(txt)
val toks = EN_PIPELINE.getTokenParser.tokenize(txt)
@@ -136,6 +137,9 @@ class NCSemanticEntityParserSpec:
require(ents.sizeIs == ids.size)
ents.map(_.getId).sorted.zip(ids.sorted).foreach { case (eId, id) =>
require(eId == id) }
+ /**
+ *
+ */
@Test
def test(): Unit =
check("t1", "t1")
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
index f8bcdee..2e40c1c 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/entity/parser/semantic/NCSemanticEntityParserYamlSpec.scala
@@ -40,6 +40,11 @@ class NCSemanticEntityParserYamlSpec:
"models/lightswitch_model.yaml"
)
+ /**
+ *
+ * @param txt
+ * @param id
+ */
private def check(txt: String, id: String): Unit =
val req = NCTestRequest(txt)
val ents = parser.parse(
@@ -54,6 +59,9 @@ class NCSemanticEntityParserYamlSpec:
require(tok.getId == id)
+ /**
+ *
+ */
@Test
def test(): Unit =
check("Turn the lights off in the entire house.", "ls:off")
\ No newline at end of file
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/opennlp/NCTestConfig.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/opennlp/NCTestConfig.scala
index 3b36d37..56eccd0 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/opennlp/NCTestConfig.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/opennlp/NCTestConfig.scala
@@ -20,6 +20,7 @@ package org.apache.nlpcraft.nlp.util.opennlp
import org.apache.nlpcraft.NCModelConfig
import org.apache.nlpcraft.nlp.token.parser.opennlp.NCOpenNlpTokenParser
import org.apache.nlpcraft.nlp.util.*
+
/**
*
*/
