This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41 by this push:
new fce8033 WIP.
fce8033 is described below
commit fce8033b1f93f81e708f50dc5e2db8d2ca58724f
Author: Aaron Radzinski <[email protected]>
AuthorDate: Mon Sep 7 16:16:29 2020 -0700
WIP.
---
.../common/inspections/NCInspectionService.scala | 51 +-------------
.../apache/nlpcraft/model/impl/NCTokenImpl.scala | 10 +--
.../model/intent/impl/NCIntentScanner.scala | 8 +--
.../nlpcraft/probe/mgrs/NCModelDecorator.scala | 36 +++++-----
.../probe/mgrs/conn/NCConnectionManager.scala | 2 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 4 --
.../inspectors/NCIntentsInspection.scala | 6 +-
.../inspectors/NCMacrosInspection.scala | 4 +-
.../inspectors/NCProbeInspection.scala} | 81 ++++++++--------------
.../inspectors/NCSynonymsInspection.scala | 4 +-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 32 ++++-----
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 20 +++---
.../dictionary/NCDictionaryEnricher.scala | 2 +-
.../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 2 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 28 ++++----
.../enrichers/relation/NCRelationEnricher.scala | 2 +-
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 2 +-
.../enrichers/stopword/NCStopWordEnricher.scala | 8 +--
.../suspicious/NCSuspiciousNounsEnricher.scala | 4 +-
.../mgrs/nlp/validate/NCValidateManager.scala | 8 +--
20 files changed, 119 insertions(+), 195 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
index aefd7aa..0222e20 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
@@ -4,12 +4,8 @@ import java.util.concurrent.{ExecutorService, Executors}
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.util.NCUtils
-import org.apache.nlpcraft.model.NCModel
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-import scala.collection.mutable
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
/**
@@ -30,52 +26,7 @@ trait NCInspectionService extends NCService {
* @param parent
* @return
*/
- def inspect(mdlId: String, inspName: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult] =
- startScopedSpan(
- "inspect",
- parent,
- "modelId" → mdlId,
- "inspName" -> inspName) { _ ⇒
- Future {
- val now = System.currentTimeMillis()
-
- val errs = mutable.Buffer.empty[String]
- val warns = mutable.Buffer.empty[String]
- val suggs = mutable.Buffer.empty[String]
-
- NCModelManager.getModel(mdlId) match {
- case Some(x) => bodyOnProbe(x.model, args, suggs, warns,
errs)
- case None => errs += s"Model not found: $mdlId"
- }
-
- NCInspectionResultImpl(
- inspectionId = inspName,
- modelId = mdlId,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- warnings = warns,
- suggestions = suggs,
- errors = errs
- )
- }(getExecutor)
- }
-
- /**
- * Convenient adapter for the probe-side inspection implementation.
- *
- * @param mdl
- * @param args
- * @param suggs Mutable collector for suggestions.
- * @param warns Mutable collector for warnings.
- * @param errs Mutable collector for errors.
- */
- protected def bodyOnProbe(
- mdl: NCModel,
- args: Option[String],
- suggs: mutable.Buffer[String],
- warns: mutable.Buffer[String],
- errs: mutable.Buffer[String]
- ) = {}
+ def inspect(mdlId: String, inspName: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
/**
*
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala
index 52d1d8d..6970e8b 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala
@@ -120,9 +120,9 @@ private[nlpcraft] object NCTokenImpl {
usrNotes.headOption match {
case Some(usrNote) ⇒
- require(mdl.elements.contains(usrNote.noteType), s"Element is
not found: ${usrNote.noteType}")
+ require(mdl.elms.contains(usrNote.noteType), s"Element is not
found: ${usrNote.noteType}")
- val elm = mdl.elements(usrNote.noteType)
+ val elm = mdl.elms(usrNote.noteType)
val ancestors = mutable.ArrayBuffer.empty[String]
var prntId = elm.getParentId
@@ -131,7 +131,7 @@ private[nlpcraft] object NCTokenImpl {
ancestors += prntId
prntId = mdl.
- elements.
+ elms.
getOrElse(prntId, throw new AssertionError(s"Element
not found: $prntId")).
getParentId
}
@@ -142,7 +142,7 @@ private[nlpcraft] object NCTokenImpl {
elm.getMetadata.asScala.foreach { case (k, v) ⇒ md.put(k,
v.asInstanceOf[java.io.Serializable]) }
new NCTokenImpl(
- mdl.model,
+ mdl.wrapper,
srvReqId = srvReqId,
id = elm.getId,
grps = elm.getGroups.asScala,
@@ -165,7 +165,7 @@ private[nlpcraft] object NCTokenImpl {
md.put("nlpcraft:nlp:freeword", !isStop && note.isNlp)
new NCTokenImpl(
- mdl.model,
+ mdl.wrapper,
srvReqId = srvReqId,
id = note.noteType, // Use NLP note type as synthetic
element ID.
grps = Seq(note.noteType), // Use NLP note type as
synthetic element group.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
index ae4b44d..b55e8c0 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
@@ -452,7 +452,7 @@ object NCIntentScanner extends LazyLogging {
if (smpAnn != null) {
if (intAnn == null && refAnn == null) {
warns += s"@NCTestSample annotation without
corresponding @NCIntent or @NCIntentRef annotations " +
- s"in method (ignoring): $mkMethodName"
+ s"in callback: $mkMethodName"
None
}
@@ -460,7 +460,7 @@ object NCIntentScanner extends LazyLogging {
val samples = smpAnn.value().toList
if (samples.isEmpty) {
- warns += s"@NCTestSample annotation is empty
in method (ignoring): $mkMethodName"
+ warns += s"@NCTestSample annotation is empty
in callback: $mkMethodName"
None
}
@@ -469,7 +469,7 @@ object NCIntentScanner extends LazyLogging {
}
}
else {
- warns += s"@NCTestSample annotation is missing in
method (ignoring): $mkMethodName"
+ warns += s"@NCTestSample annotation is missing in
callback: $mkMethodName"
None
}
@@ -479,7 +479,7 @@ object NCIntentScanner extends LazyLogging {
}).toMap
if (!annFound)
- warns += s"Model '${mdl.getId}' doesn't have any intents."
+ warns += s"No intents found."
NCIntentSamplesScanResult(res, warns)
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
index 3763f33..f1a5a6f 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
@@ -23,7 +23,7 @@ import java.util
import org.apache.nlpcraft.common.TOK_META_ALIASES_KEY
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.model.impl.{NCModelWrapper, NCTokenImpl,
NCVariantImpl}
-import org.apache.nlpcraft.model.{NCElement, NCModel, NCVariant}
+import org.apache.nlpcraft.model.{NCElement, NCVariant}
import scala.collection.JavaConverters._
import scala.collection.{Seq, mutable}
@@ -31,22 +31,22 @@ import scala.language.implicitConversions
/**
*
- * @param model Decorated model.
- * @param synonyms Fast-access synonyms map for first phase.
- * @param synonymsDsl Fast-access synonyms map for second phase.
- * @param additionalStopWordsStems Stemmatized additional stopwords.
- * @param excludedStopWordsStems Stemmatized excluded stopwords.
- * @param suspiciousWordsStems Stemmatized suspicious stopwords.
- * @param elements Map of model elements.
+ * @param wrapper Decorated model.
+ * @param syns Fast-access synonyms map for first phase.
+ * @param synsDsl Fast-access synonyms map for second phase.
+ * @param addStopWordsStems Stemmatized additional stopwords.
+ * @param exclStopWordsStems Stemmatized excluded stopwords.
+ * @param suspWordsStems Stemmatized suspicious stopwords.
+ * @param elms Map of model elements.
*/
case class NCModelDecorator(
- model: NCModelWrapper,
- synonyms: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
- synonymsDsl: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
- additionalStopWordsStems: Set[String],
- excludedStopWordsStems: Set[String],
- suspiciousWordsStems: Set[String],
- elements: Map[String/*Element ID*/, NCElement]
+ wrapper: NCModelWrapper,
+ syns: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
+ synsDsl: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
+ addStopWordsStems: Set[String],
+ exclStopWordsStems: Set[String],
+ suspWordsStems: Set[String],
+ elms: Map[String/*Element ID*/, NCElement]
) extends java.io.Serializable {
/**
* Makes variants for given sentences.
@@ -109,9 +109,9 @@ case class NCModelDecorator(
override def toString: String = {
s"Probe model decorator [" +
- s"id=${model.getId}, " +
- s"name=${model.getName}, " +
- s"version=${model.getVersion}" +
+ s"id=${wrapper.getId}, " +
+ s"name=${wrapper.getName}, " +
+ s"version=${wrapper.getVersion}" +
s"]"
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
index 96bb4c2..dafaf5f 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
@@ -229,7 +229,7 @@ object NCConnectionManager extends NCService {
"PROBE_HW_ADDR" → hwAddrs,
"PROBE_MODELS" →
NCModelManager.getAllModels().map(m ⇒ {
- val mdl = m.model
+ val mdl = m.wrapper
// Model already validated.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index e696bde..3aca836 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -39,10 +39,6 @@ import scala.util.control.Exception._
* Model deployment manager.
*/
object NCDeployManager extends NCService with DecorateAsScala {
- private final val CLS_SAMPLE = classOf[NCIntentSample]
- private final val CLS_INTENT = classOf[NCIntent]
- private final val CLS_INTENT_REF = classOf[NCIntentRef]
-
private final val ID_REGEX = "^[_a-zA-Z]+[a-zA-Z0-9:-_]*$"
@volatile private var models: ArrayBuffer[NCModelWrapper] = _
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
index 592a606..009be47 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
@@ -29,12 +29,12 @@ import scala.collection._
/**
* Inspection for model's intents.
*/
-object NCIntentsInspection extends NCInspectionService {
+object NCIntentsInspection extends NCProbeInspection {
private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
override def getName: String = "intents"
- override def bodyOnProbe(
+ override def body(
mdl: NCModel,
args: Option[String],
suggs: mutable.Buffer[String],
@@ -64,7 +64,7 @@ object NCIntentsInspection extends NCInspectionService {
val seq: Seq[String] = sNorm.split("
").map(NCNlpPorterStemmer.stem)
if (!allSyns.exists(_.intersect(seq).nonEmpty))
- warns += s"Sample: '$s' doesn't contain synonyms"
+ warns += s"Intent sample '$s' doesn't contain any
direct synonyms."
}
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
index 713343b..005f520 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
@@ -26,10 +26,10 @@ import scala.collection.mutable
/**
* Inspection for model's macros.
*/
-object NCMacrosInspection extends NCInspectionService {
+object NCMacrosInspection extends NCProbeInspection {
override def getName: String = "macros"
- override def bodyOnProbe(
+ override def body(
mdl: NCModel,
args: Option[String],
suggs: mutable.Buffer[String],
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
similarity index 53%
copy from
nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
copy to
nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
index aefd7aa..6c856cd 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
@@ -1,27 +1,32 @@
-package org.apache.nlpcraft.common.inspections
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.concurrent.{ExecutorService, Executors}
+package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.common.util.NCUtils
+import org.apache.nlpcraft.common.inspections.{NCInspectionResult,
NCInspectionService}
import org.apache.nlpcraft.model.NCModel
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.mutable
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
-
-/**
- * Base trait for inspection implementation.
- */
-trait NCInspectionService extends NCService {
- /** */
- @volatile private var pool: ExecutorService = _
-
- /** */
- @volatile private var executor: ExecutionContextExecutor = _
+import scala.concurrent.Future
+trait NCProbeInspection extends NCInspectionService {
/**
*
* @param mdlId
@@ -30,21 +35,22 @@ trait NCInspectionService extends NCService {
* @param parent
* @return
*/
- def inspect(mdlId: String, inspName: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult] =
+ override def inspect(mdlId: String, inspName: String, args:
Option[String], parent: Span = null): Future[NCInspectionResult] = {
+ val now = System.currentTimeMillis()
+
startScopedSpan(
"inspect",
parent,
"modelId" → mdlId,
- "inspName" -> inspName) { _ ⇒
+ "inspName" -> inspName,
+ "args" -> args.orNull) { _ ⇒
Future {
- val now = System.currentTimeMillis()
-
val errs = mutable.Buffer.empty[String]
val warns = mutable.Buffer.empty[String]
val suggs = mutable.Buffer.empty[String]
NCModelManager.getModel(mdlId) match {
- case Some(x) => bodyOnProbe(x.model, args, suggs, warns,
errs)
+ case Some(x) => body(x.wrapper, args, suggs, warns, errs)
case None => errs += s"Model not found: $mdlId"
}
@@ -59,6 +65,7 @@ trait NCInspectionService extends NCService {
)
}(getExecutor)
}
+ }
/**
* Convenient adapter for the probe-side inspection implementation.
@@ -69,41 +76,11 @@ trait NCInspectionService extends NCService {
* @param warns Mutable collector for warnings.
* @param errs Mutable collector for errors.
*/
- protected def bodyOnProbe(
+ protected def body(
mdl: NCModel,
args: Option[String],
suggs: mutable.Buffer[String],
warns: mutable.Buffer[String],
errs: mutable.Buffer[String]
- ) = {}
-
- /**
- *
- * @return
- */
- def getExecutor = executor
-
- /**
- *
- * @return
- */
- def getName: String
-
- override def start(parent: Span): NCService =
- startScopedSpan("start", parent) { _ ⇒
- pool = Executors.newCachedThreadPool()
- executor = ExecutionContext.fromExecutor(pool)
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit =
- startScopedSpan("stop", parent) { _ ⇒
- super.stop(parent)
-
- NCUtils.shutdownPools(pool)
-
- pool = null
- executor = null
- }
+ )
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
index bc0d92c..77575af 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
@@ -27,14 +27,14 @@ import scala.collection.mutable
/**
* Inspection for model's synonyms.
*/
-object NCSynonymsInspection extends NCInspectionService {
+object NCSynonymsInspection extends NCProbeInspection {
// Pretty arbitrary number...
// TODO: make it part of inspection configuration.
private final val TOO_MANY_SYNS = 10000
override def getName: String = "synonyms"
- override def bodyOnProbe(
+ override def body(
mdl: NCModel,
args: Option[String],
suggs: mutable.Buffer[String],
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index e1dd31d..f233a95 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -99,13 +99,13 @@ object NCModelManager extends NCService with
DecorateAsScala {
val tbl = NCAsciiTable("Model ID", "Name", "Ver.", "Elements",
"Synonyms")
models.values.foreach(mdl ⇒ {
- val synCnt = mdl.synonyms.values.flatMap(_.values).flatten.size
+ val synCnt = mdl.syns.values.flatMap(_.values).flatten.size
tbl += (
- mdl.model.getId,
- mdl.model.getName,
- mdl.model.getVersion,
- mdl.elements.keySet.size,
+ mdl.wrapper.getId,
+ mdl.wrapper.getName,
+ mdl.wrapper.getVersion,
+ mdl.elms.keySet.size,
synCnt
)
@@ -114,7 +114,7 @@ object NCModelManager extends NCService with
DecorateAsScala {
tbl.info(logger, Some(s"Models deployed: ${models.size}\n"))
for (mdl ← models.values; insId ← DFLT_INSPECTIONS) {
- val mdlId = mdl.model.getId
+ val mdlId = mdl.wrapper.getId
NCInspectionManager.inspect(mdlId, insId, null,
parent).onComplete{
case Success(res) =>
@@ -134,7 +134,7 @@ object NCModelManager extends NCService with
DecorateAsScala {
addTags(
span,
- "deployedModels" →
models.values.map(_.model.getId).mkString(",")
+ "deployedModels" →
models.values.map(_.wrapper.getId).mkString(",")
)
}
@@ -162,7 +162,7 @@ object NCModelManager extends NCService with
DecorateAsScala {
override def stop(parent: Span = null): Unit = startScopedSpan("stop",
parent) { _ ⇒
mux.synchronized {
if (models != null)
- models.values.foreach(m ⇒ discardModel(m.model))
+ models.values.foreach(m ⇒ discardModel(m.wrapper))
}
super.stop()
@@ -552,13 +552,13 @@ object NCModelManager extends NCService with
DecorateAsScala {
})
NCModelDecorator(
- model = mdl,
- synonyms = mkFastAccessMap(filter(syns, dsl = false)),
- synonymsDsl = mkFastAccessMap(filter(syns, dsl = true)),
- additionalStopWordsStems = addStopWords,
- excludedStopWordsStems = exclStopWords,
- suspiciousWordsStems = suspWords,
- elements = mdl.getElements.map(elm ⇒ (elm.getId, elm)).toMap
+ wrapper = mdl,
+ syns = mkFastAccessMap(filter(syns, dsl = false)),
+ synsDsl = mkFastAccessMap(filter(syns, dsl = true)),
+ addStopWordsStems = addStopWords,
+ exclStopWordsStems = exclStopWords,
+ suspWordsStems = suspWords,
+ elms = mdl.getElements.map(elm ⇒ (elm.getId, elm)).toMap
)
}
@@ -763,7 +763,7 @@ object NCModelManager extends NCService with
DecorateAsScala {
def getModelInfo(mdlId: String, parent: Span = null):
java.util.Map[String, Any] =
startScopedSpan("getModel", parent, "mdlId" → mdlId) { _ ⇒
val mdl = mux.synchronized { models.get(mdlId) }.
- getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
+ getOrElse(throw new NCE(s"Model not found: '$mdlId'")).wrapper
val data = new util.HashMap[String, Any]()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index 9016cb3..acea56c 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -368,7 +368,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
case class Holder(enricher: NCProbeEnricher, getNotes: () ⇒
Seq[NCNlpSentenceNote])
def get(name: String, e: NCProbeEnricher): Option[Holder] =
- if (mdlDec.model.getEnabledBuiltInTokens.contains(name))
+ if (mdlDec.wrapper.getEnabledBuiltInTokens.contains(name))
Some(Holder(e, () ⇒ nlpSen.flatten.filter(_.noteType ==
name)))
else
None
@@ -493,7 +493,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
val fltSenVars: Seq[(NCVariant, Int)] =
senVars.
zipWithIndex.
- flatMap { case (variant, i) ⇒ if
(mdlDec.model.onParsedVariant(variant)) Some(variant, i) else None }
+ flatMap { case (variant, i) ⇒ if
(mdlDec.wrapper.onParsedVariant(variant)) Some(variant, i) else None }
senVars = fltSenVars.map(_._1)
val allVars = senVars.flatMap(_.asScala)
@@ -528,7 +528,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
// Create model query context.
val ctx: NCContext = new NCContext {
override lazy val getRequest: NCRequest = req
- override lazy val getModel: NCModel = mdlDec.model
+ override lazy val getModel: NCModel = mdlDec.wrapper
override lazy val getServerRequestId: String = srvReqId
override lazy val getConversation: NCConversation = new
NCConversation {
@@ -546,7 +546,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
logKey = U.mkLogHolderKey(srvReqId)
- val meta = mdlDec.model.getMetadata
+ val meta = mdlDec.wrapper.getMetadata
meta.synchronized {
meta.put(logKey, logHldr)
@@ -572,19 +572,19 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
def onFinish(): Unit = {
if (logKey != null)
- mdlDec.model.getMetadata.remove(logKey)
+ mdlDec.wrapper.getMetadata.remove(logKey)
span.end()
}
- val mdl: NCModelWrapper = mdlDec.model
+ val mdl: NCModelWrapper = mdlDec.wrapper
val solverIn = new NCIntentSolverInput(ctx)
// Execute model query asynchronously.
U.asFuture(
_ ⇒ {
- var res = mdlDec.model.onContext(ctx)
+ var res = mdlDec.wrapper.onContext(ctx)
start = System.currentTimeMillis()
@@ -627,7 +627,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
if (e.getCause != null)
logger.info(s"Rejection cause:", e.getCause)
- val res =
mdlDec.model.onRejection(solverIn.intentMatch, e)
+ val res =
mdlDec.wrapper.onRejection(solverIn.intentMatch, e)
if (res != null)
respondWithResult(res, None)
@@ -656,7 +656,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
logger.error(s"Unexpected error for server request ID:
$srvReqId", e)
- val res = mdlDec.model.onError(ctx, e)
+ val res = mdlDec.wrapper.onError(ctx, e)
if (res != null)
respondWithResult(res, None)
@@ -682,7 +682,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
"resBody" → res.getBody
)
- val res0 = mdlDec.model.onResult(solverIn.intentMatch, res)
+ val res0 = mdlDec.wrapper.onResult(solverIn.intentMatch,
res)
respondWithResult(if (res0 != null) res0 else res, if
(logHldr != null) Some(logHldr.toJson) else None)
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala
index b2c4137..b1cd2fa 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala
@@ -57,7 +57,7 @@ object NCDictionaryEnricher extends NCProbeEnricher {
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta:
Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
ns.foreach(t ⇒ {
// Dictionary.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
index 9e4d34f..7b583d6 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
@@ -239,7 +239,7 @@ object NCLimitEnricher extends NCProbeEnricher {
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta:
Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
val notes = mutable.HashSet.empty[NCNlpSentenceNote]
val numsMap =
NCNumericManager.find(ns).filter(_.unit.isEmpty).map(p ⇒ p.tokens → p).toMap
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 9ae77f9..dcd71a2 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -297,15 +297,15 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
*/
private def alreadyMarked(toks: Seq[NCNlpSentenceToken], elemId: String):
Boolean = toks.forall(_.isTypeOf(elemId))
- def isComplex(mdl: NCModelDecorator): Boolean = mdl.synonymsDsl.nonEmpty
|| !mdl.model.getParsers.isEmpty
+ def isComplex(mdl: NCModelDecorator): Boolean = mdl.synsDsl.nonEmpty ||
!mdl.wrapper.getParsers.isEmpty
@throws[NCE]
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta:
Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { span ⇒
- val jiggleFactor = mdl.model.getJiggleFactor
+ val jiggleFactor = mdl.wrapper.getJiggleFactor
val cache = mutable.HashSet.empty[Seq[Int]]
val matches = ArrayBuffer.empty[ElementMatch]
@@ -352,7 +352,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
var seq: Seq[Seq[Complex]] = null
// Attempt to match each element.
- for (elm ← mdl.elements.values if !alreadyMarked(toks,
elm.getId)) {
+ for (elm ← mdl.elms.values if !alreadyMarked(toks,
elm.getId)) {
var found = false
def addMatch(
@@ -365,12 +365,12 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
}
// Optimization - plain synonyms can be used only
on first iteration
- if (mdl.synonyms.nonEmpty && !ns.exists(_.isUser))
- for (syn ← fastAccess(mdl.synonyms, elm.getId,
toks.length) if !found)
+ if (mdl.syns.nonEmpty && !ns.exists(_.isUser))
+ for (syn ← fastAccess(mdl.syns, elm.getId,
toks.length) if !found)
if (syn.isMatch(toks))
addMatch(elm, toks, syn, Seq.empty)
- if (mdl.synonymsDsl.nonEmpty) {
+ if (mdl.synsDsl.nonEmpty) {
found = false
if (collapsedSens == null)
@@ -379,7 +379,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
if (seq == null)
seq = convert(ns, collapsedSens, toks)
- for (comb ← seq; syn ←
fastAccess(mdl.synonymsDsl, elm.getId, comb.length) if !found)
+ for (comb ← seq; syn ← fastAccess(mdl.synsDsl,
elm.getId, comb.length) if !found)
if (syn.isMatch(comb.map(_.data)))
addMatch(elm, toks, syn,
comb.filter(_.isToken).map(_.token))
}
@@ -392,7 +392,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
startScopedSpan("jiggleProc", span,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
// Iterate over depth-limited permutations of the original
sentence with and without stopwords.
jiggle(ns, jiggleFactor).foreach(procPerm)
@@ -413,7 +413,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
for ((m, idx) ← matches.zipWithIndex) {
if (DEEP_DEBUG)
logger.trace(
- s"Model '${mdl.model.getId}' element found (${idx + 1}
of $matchCnt) [" +
+ s"Model '${mdl.wrapper.getId}' element found (${idx +
1} of $matchCnt) [" +
s"elementId=${m.element.getId}, " +
s"synonym=${m.synonym}, " +
s"tokens=${tokString(m.tokens)}" +
@@ -429,14 +429,14 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
mark(ns, elem = elm, toks = m.tokens, direct = direct, syn =
Some(syn), metaOpt = None, parts = m.parts)
}
- val parsers = mdl.model.getParsers
+ val parsers = mdl.wrapper.getParsers
for (parser ← parsers.asScala) {
parser.onInit()
startScopedSpan("customParser", span,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
def to(t: NCNlpSentenceToken): NCCustomWord =
new NCCustomWord {
@@ -458,7 +458,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
val res = parser.parse(
NCRequestImpl(senMeta, ns.srvReqId),
- mdl.model,
+ mdl.wrapper,
ns.map(to).asJava,
ns.flatten.distinct.filter(!_.isNlp).map(n ⇒ {
val noteId = n.noteType
@@ -494,7 +494,7 @@ object NCModelEnricher extends NCProbeEnricher with
DecorateAsScala {
if (!alreadyMarked(matchedToks, elemId))
mark(
ns,
- elem = mdl.elements.getOrElse(elemId,
throw new NCE(s"Custom model parser returned unknown element ID: $elemId")),
+ elem = mdl.elms.getOrElse(elemId, throw
new NCE(s"Custom model parser returned unknown element ID: $elemId")),
toks = matchedToks,
direct = true,
syn = None,
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
index 0318533..da439d4 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
@@ -141,7 +141,7 @@ object NCRelationEnricher extends NCProbeEnricher {
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta:
Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
// Tries to grab tokens direct way.
// Example: A, B, C ⇒ ABC, AB, BC .. (AB will be processed first)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
index 30954cd..9021c96 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
@@ -418,7 +418,7 @@ object NCSortEnricher extends NCProbeEnricher {
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, meta:
Map[String, Serializable], parent: Span): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
val notes = mutable.HashSet.empty[NCNlpSentenceNote]
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala
index d366484..089b5ff 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala
@@ -189,7 +189,7 @@ object NCStopWordEnricher extends NCProbeEnricher {
(tok, idx) ← ns.zipWithIndex
if idx != max &&
!tok.isStopWord &&
- !mdl.excludedStopWordsStems.contains(tok.stem) &&
+ !mdl.exclStopWordsStems.contains(tok.stem) &&
POSES.contains(tok.pos) &&
ns(idx + 1).isStopWord
) {
@@ -210,10 +210,10 @@ object NCStopWordEnricher extends NCProbeEnricher {
def mark(stems: Set[String], f: Boolean): Unit =
ns.filter(t ⇒ stems.contains(t.stem)).foreach(t ⇒
ns.fixNote(t.getNlpNote, "stopWord" → f))
- startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, "modelId"
→ mdl.model.getId, "txt" → ns.text) { _ ⇒
+ startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, "modelId"
→ mdl.wrapper.getId, "txt" → ns.text) { _ ⇒
- mark(mdl.excludedStopWordsStems, f = false)
- mark(mdl.additionalStopWordsStems, f = true)
+ mark(mdl.exclStopWordsStems, f = false)
+ mark(mdl.addStopWordsStems, f = true)
processGeo(ns)
processDate(ns)
processNums(ns)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala
index be05916..de7799d 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala
@@ -43,8 +43,8 @@ object NCSuspiciousNounsEnricher extends NCProbeEnricher {
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta:
Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
- "modelId" → mdl.model.getId,
+ "modelId" → mdl.wrapper.getId,
"txt" → ns.text) { _ ⇒
- ns.filter(t ⇒ mdl.suspiciousWordsStems.contains(t.stem)).foreach(t
⇒ ns.fixNote(t.getNlpNote, "suspNoun" → true))
+ ns.filter(t ⇒ mdl.suspWordsStems.contains(t.stem)).foreach(t ⇒
ns.fixNote(t.getNlpNote, "suspNoun" → true))
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala
index 76fc21a..42bce81 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala
@@ -55,8 +55,8 @@ object NCValidateManager extends NCService with LazyLogging {
startScopedSpan("validate", parent,
"srvReqId" → ns.srvReqId,
"txt" → ns.text,
- "modelId" → mdl.model.getId) { _ ⇒
- val model = mdl.model
+ "modelId" → mdl.wrapper.getId) { _ ⇒
+ val model = mdl.wrapper
if (!model.isNotLatinCharsetAllowed &&
!ns.text.matches("""[\s\w\p{Punct}]+"""))
throw NCValidateException("ALLOW_NON_LATIN_CHARSET")
@@ -81,10 +81,10 @@ object NCValidateManager extends NCService with LazyLogging
{
startScopedSpan("validate", parent,
"srvReqId" → ns.srvReqId,
"txt" → ns.text,
- "modelId" → mdl.model.getId) { _ ⇒
+ "modelId" → mdl.wrapper.getId) { _ ⇒
val types = ns.flatten.filter(!_.isNlp).map(_.noteType).distinct
val overlapNotes = ns.map(tkn ⇒ types.flatMap(tp ⇒
tkn.getNotes(tp))).filter(_.size > 1).flatten
- val model = mdl.model
+ val model = mdl.wrapper
if (overlapNotes.nonEmpty)
throw NCValidateException("OVERLAP_NOTES")
