This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-41-1 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit f8f600b40d9a47a7d7ee72bc8727fd1fe9ce57ce Author: Sergey Kamov <[email protected]> AuthorDate: Wed Sep 9 14:50:45 2020 +0300 WIP. --- .../nlpcraft/model/impl/NCModelWrapper.scala | 151 --------------------- .../apache/nlpcraft/model/impl/NCTokenImpl.scala | 11 +- .../model/intent/impl/NCIntentSolver.scala | 6 +- .../probe/mgrs/conn/NCConnectionManager.scala | 3 +- .../probe/mgrs/deploy/NCDeployManager.scala | 35 ++--- .../inspections/inspectors/NCProbeInspection.scala | 4 +- .../nlpcraft/probe/mgrs/model/NCModelManager.scala | 40 +++--- .../nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala | 2 +- .../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 44 +++--- .../dictionary/NCDictionaryEnricher.scala | 4 +- .../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 4 +- .../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 38 +++--- .../enrichers/relation/NCRelationEnricher.scala | 4 +- .../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 4 +- .../enrichers/stopword/NCStopWordEnricher.scala | 4 +- .../suspicious/NCSuspiciousNounsEnricher.scala | 4 +- .../mgrs/nlp/validate/NCValidateManager.scala | 16 ++- 17 files changed, 114 insertions(+), 260 deletions(-) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala deleted file mode 100644 index c356f90..0000000 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nlpcraft.model.impl - -import java.io.Serializable -import java.util - -import org.apache.nlpcraft.common.TOK_META_ALIASES_KEY -import org.apache.nlpcraft.common.nlp.NCNlpSentence -import org.apache.nlpcraft.model.intent.impl.NCIntentSolver -import org.apache.nlpcraft.model.{NCContext, NCCustomParser, NCElement, NCIntentMatch, NCModel, NCRejection, NCResult, NCVariant} -import org.apache.nlpcraft.probe.mgrs.NCSynonym - -import scala.collection.JavaConverters._ -import scala.collection.{Seq, mutable} - -/** - * - * @param proxy - * @param solver - * @param syns - * @param synsDsl - * @param addStopWordsStems - * @param exclStopWordsStems - * @param suspWordsStems - * @param elms - */ -case class NCModelWrapper( - proxy: NCModel, - solver: NCIntentSolver, - syns: Map[String/*Element ID*/, Map[Int/*Synonym length*/, Seq[NCSynonym]]], // Fast access map. - synsDsl: Map[String/*Element ID*/, Map[Int/*Synonym length*/, Seq[NCSynonym]]], // Fast access map. - addStopWordsStems: Set[String], - exclStopWordsStems: Set[String], - suspWordsStems: Set[String], - elms: Map[String/*Element ID*/, NCElement] -) extends NCModel { - require(proxy != null) - - override def getId: String = proxy.getId - override def getName: String = proxy.getName - override def getVersion: String = proxy.getVersion - override def getDescription: String = proxy.getDescription - override def getMaxUnknownWords: Int = proxy.getMaxUnknownWords - override def getMaxFreeWords: Int = proxy.getMaxFreeWords - override def getMaxSuspiciousWords: Int = proxy.getMaxSuspiciousWords - override def getMinWords: Int = proxy.getMinWords - override def getMaxWords: Int = proxy.getMaxWords - override def getMinTokens: Int = proxy.getMinTokens - override def getMaxTokens: Int = proxy.getMaxTokens - override def getMinNonStopwords: Int = proxy.getMinNonStopwords - override def isNonEnglishAllowed: Boolean = proxy.isNonEnglishAllowed - override def isNotLatinCharsetAllowed: Boolean = proxy.isNotLatinCharsetAllowed - override def isSwearWordsAllowed: Boolean = proxy.isSwearWordsAllowed - override def isNoNounsAllowed: Boolean = proxy.isNoNounsAllowed - override def isPermutateSynonyms: Boolean = proxy.isPermutateSynonyms - override def isDupSynonymsAllowed: Boolean = proxy.isDupSynonymsAllowed - override def getMaxTotalSynonyms: Int = proxy.getMaxTotalSynonyms - override def isNoUserTokensAllowed: Boolean = proxy.isNoUserTokensAllowed - override def getJiggleFactor: Int = proxy.getJiggleFactor - override def getMetadata: util.Map[String, AnyRef] = proxy.getMetadata - override def getAdditionalStopWords: util.Set[String] = proxy.getAdditionalStopWords - override def getExcludedStopWords: util.Set[String] = proxy.getExcludedStopWords - override def getSuspiciousWords: util.Set[String] = proxy.getSuspiciousWords - override def getMacros: util.Map[String, String] = proxy.getMacros - override def getParsers: util.List[NCCustomParser] = proxy.getParsers - override def getElements: util.Set[NCElement] = proxy.getElements - override def getEnabledBuiltInTokens: util.Set[String] = proxy.getEnabledBuiltInTokens - override def onParsedVariant(`var`: NCVariant): Boolean = proxy.onParsedVariant(`var`) - override def onContext(ctx: NCContext): NCResult = proxy.onContext(ctx) - override def onMatchedIntent(ctx: NCIntentMatch): Boolean = proxy.onMatchedIntent(ctx) - override def onResult(ctx: NCIntentMatch, res: NCResult): NCResult = proxy.onResult(ctx, res) - override def onRejection(ctx: NCIntentMatch, e: NCRejection): NCResult = proxy.onRejection(ctx, e) - override def onError(ctx: NCContext, e: Throwable): NCResult = proxy.onError(ctx, e) - override def onInit(): Unit = proxy.onInit() - override def onDiscard(): Unit = proxy.onDiscard() - - /** - * Makes variants for given sentences. - * - * @param srvReqId Server request ID. - * @param sens Sentences. - */ - def makeVariants(srvReqId: String, sens: Seq[NCNlpSentence]): Seq[NCVariant] = { - val seq = sens.map(_.toSeq.map(nlpTok ⇒ NCTokenImpl(this, srvReqId, nlpTok) → nlpTok)) - val toks = seq.map(_.map { case (tok, _) ⇒ tok }) - - case class Key(id: String, from: Int, to: Int) - - val keys2Toks = toks.flatten.map(t ⇒ Key(t.getId, t.getStartCharIndex, t.getEndCharIndex) → t).toMap - val partsKeys = mutable.HashSet.empty[Key] - - seq.flatten.foreach { case (tok, tokNlp) ⇒ - if (tokNlp.isUser) { - val userNotes = tokNlp.filter(_.isUser) - - require(userNotes.size == 1) - - val optList: Option[util.List[util.HashMap[String, Serializable]]] = userNotes.head.dataOpt("parts") - - optList match { - case Some(list) ⇒ - val keys = - list.asScala.map(m ⇒ - Key( - m.get("id").asInstanceOf[String], - m.get("startcharindex").asInstanceOf[Integer], - m.get("endcharindex").asInstanceOf[Integer] - ) - ) - val parts = keys.map(keys2Toks) - - parts.zip(list.asScala).foreach { case (part, map) ⇒ - map.get(TOK_META_ALIASES_KEY) match { - case null ⇒ // No-op. - case aliases ⇒ part.getMetadata.put(TOK_META_ALIASES_KEY, aliases.asInstanceOf[Object]) - } - } - - tok.setParts(parts) - partsKeys ++= keys - - case None ⇒ // No-op. - } - } - } - - // We can't collapse parts earlier, because we need them here (setParts method, few lines above.) - toks.filter(sen ⇒ - !sen.exists(t ⇒ - t.getId != "nlpcraft:nlp" && - partsKeys.contains(Key(t.getId, t.getStartCharIndex, t.getEndCharIndex)) - ) - ).map(p ⇒ new NCVariantImpl(p.asJava)) - } -} diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala index 66ab4cb..0c5dd48 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCTokenImpl.scala @@ -23,6 +23,7 @@ import java.util.Collections import org.apache.nlpcraft.common._ import org.apache.nlpcraft.common.nlp.NCNlpSentenceToken import org.apache.nlpcraft.model._ +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import scala.collection.JavaConverters._ import scala.collection.{Seq, mutable} @@ -119,9 +120,9 @@ private[nlpcraft] object NCTokenImpl { usrNotes.headOption match { case Some(usrNote) ⇒ - require(mdl.elms.contains(usrNote.noteType), s"Element is not found: ${usrNote.noteType}") + require(mdl.elements.contains(usrNote.noteType), s"Element is not found: ${usrNote.noteType}") - val elm = mdl.elms(usrNote.noteType) + val elm = mdl.elements(usrNote.noteType) val ancestors = mutable.ArrayBuffer.empty[String] var prntId = elm.getParentId @@ -130,7 +131,7 @@ private[nlpcraft] object NCTokenImpl { ancestors += prntId prntId = mdl. - elms. + elements. getOrElse(prntId, throw new AssertionError(s"Element not found: $prntId")). getParentId } @@ -141,7 +142,7 @@ private[nlpcraft] object NCTokenImpl { elm.getMetadata.asScala.foreach { case (k, v) ⇒ md.put(k, v.asInstanceOf[java.io.Serializable]) } new NCTokenImpl( - mdl, + mdl.proxy, srvReqId = srvReqId, id = elm.getId, grps = elm.getGroups.asScala, @@ -164,7 +165,7 @@ private[nlpcraft] object NCTokenImpl { md.put("nlpcraft:nlp:freeword", !isStop && note.isNlp) new NCTokenImpl( - mdl, + mdl.proxy, srvReqId = srvReqId, id = note.noteType, // Use NLP note type as synthetic element ID. grps = Seq(note.noteType), // Use NLP note type as synthetic element group. diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala index 9d6686a..4dbedc4 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala @@ -23,9 +23,9 @@ import org.apache.nlpcraft.common.NCException import org.apache.nlpcraft.common.debug.NCLogHolder import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace import org.apache.nlpcraft.common.util.NCUtils -import org.apache.nlpcraft.model.impl.{NCModelWrapper, NCVariantImpl} -import org.apache.nlpcraft.model.{NCContext, NCIntentMatch, NCIntentSkip, NCRejection, NCResult, NCToken, NCVariant} +import org.apache.nlpcraft.model.impl.NCVariantImpl import org.apache.nlpcraft.model.intent.utils.NCDslIntent +import org.apache.nlpcraft.model.{NCContext, NCIntentMatch, NCIntentSkip, NCModel, NCRejection, NCResult, NCToken, NCVariant} import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager import scala.collection.JavaConverters._ @@ -126,7 +126,7 @@ class NCIntentSolver(intents: List[(NCDslIntent/*Intent*/, NCIntentMatch ⇒ NCR res.groups.find(_.termId == termId).flatMap(grp ⇒ Some(grp.tokens)).getOrElse(Nil).asJava } - if (!in.context.getModel.asInstanceOf[NCModelWrapper].onMatchedIntent(intentMatch)) { + if (!in.context.getModel.asInstanceOf[NCModel].onMatchedIntent(intentMatch)) { logger.info( s"Model '${ctx.getModel.getId}' triggered rematching of intents " + s"by intent '${res.intentId}' on variant #${res.variantIdx + 1}." diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala index ab24173..5a8c290 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala @@ -227,7 +227,8 @@ object NCConnectionManager extends NCService { "PROBE_HOST_ADDR" → localHost.getHostAddress, "PROBE_HW_ADDR" → hwAddrs, "PROBE_MODELS" → - NCModelManager.getAllModels().map(mdl ⇒ { + NCModelManager.getAllModelWrappers().map(wrapper ⇒ { + val mdl = wrapper.proxy // Model already validated. diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala index 8c10c1d..2fb3e52 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala @@ -29,10 +29,9 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.util.NCUtils.{DSL_FIX, REGEX_FIX} import org.apache.nlpcraft.model._ import org.apache.nlpcraft.model.factories.basic.NCBasicModelFactory -import org.apache.nlpcraft.model.impl.NCModelWrapper import org.apache.nlpcraft.model.intent.impl.{NCIntentScanner, NCIntentSolver} import org.apache.nlpcraft.probe.mgrs.NCSynonymChunkKind.{DSL, REGEX, TEXT} -import org.apache.nlpcraft.probe.mgrs.{NCSynonym, NCSynonymChunk} +import org.apache.nlpcraft.probe.mgrs.{NCSynonym, NCSynonymChunk, deploy} import org.apache.nlpcraft.probe.mgrs.model.NCModelSynonymDslCompiler import resource.managed @@ -49,7 +48,7 @@ object NCDeployManager extends NCService with DecorateAsScala { private final val TOKENS_PROVIDERS_PREFIXES = Set("nlpcraft:", "google:", "stanford:", "opennlp:", "spacy:") private final val ID_REGEX = "^[_a-zA-Z]+[a-zA-Z0-9:-_]*$" - @volatile private var models: ArrayBuffer[NCModelWrapper] = _ + @volatile private var wrappers: ArrayBuffer[NCModelWrapper] = _ @volatile private var modelFactory: NCModelFactory = _ object Config extends NCConfigurable { @@ -143,7 +142,8 @@ object NCDeployManager extends NCService with DecorateAsScala { val exclStopWords = checkAndStemmatize(mdl.getExcludedStopWords, "Excluded stopword") val suspWords = checkAndStemmatize(mdl.getSuspiciousWords, "Suspicious word") - checkStopwordsDups(addStopWords, exclStopWords) + // TODO: skh + //checkStopwordsDups(addStopWords, exclStopWords) val syns = mutable.HashSet.empty[SynonymHolder] @@ -376,15 +376,15 @@ object NCDeployManager extends NCService with DecorateAsScala { mdl.getEnabledBuiltInTokens.asScala ) - NCModelWrapper( + deploy.NCModelWrapper( proxy = mdl, solver = solver, - syns = mkFastAccessMap(filter(syns, dsl = false)), - synsDsl = mkFastAccessMap(filter(syns, dsl = true)), + synonyms = mkFastAccessMap(filter(syns, dsl = false)), + synonymsDsl = mkFastAccessMap(filter(syns, dsl = true)), addStopWordsStems = addStopWords, exclStopWordsStems = exclStopWords, suspWordsStems = suspWords, - elms = mdl.getElements.asScala.map(elm ⇒ (elm.getId, elm)).toMap + elements = mdl.getElements.asScala.map(elm ⇒ (elm.getId, elm)).toMap ) } @@ -419,7 +419,7 @@ object NCDeployManager extends NCService with DecorateAsScala { * @param clsName Model class name. */ @throws[NCE] - private def makeModel(clsName: String): NCModelWrapper = + private def makeModelWrapper(clsName: String): NCModelWrapper = try wrap( makeModelFromSource( @@ -515,7 +515,7 @@ object NCDeployManager extends NCService with DecorateAsScala { @throws[NCE] override def start(parent: Span = null): NCService = startScopedSpan("start", parent) { _ ⇒ modelFactory = new NCBasicModelFactory - models = ArrayBuffer.empty[NCModelWrapper] + wrappers = ArrayBuffer.empty[NCModelWrapper] // Initialize model factory (if configured). Config.modelFactoryType match { @@ -527,7 +527,7 @@ object NCDeployManager extends NCService with DecorateAsScala { case None ⇒ // No-op. } - models ++= Config.models.map(makeModel) + wrappers ++= Config.models.map(makeModelWrapper) Config.jarsFolder match { case Some(jarsFolder) ⇒ @@ -542,13 +542,14 @@ object NCDeployManager extends NCService with DecorateAsScala { val locJar = if (src == null) null else new File(src.getLocation.getPath) for (jar ← scanJars(jarsFile) if jar != locJar) - models ++= extractModels(jar) + wrappers ++= extractModels(jar) case None ⇒ // No-op. } // Verify models' identities. - models.foreach(mdl ⇒ { + wrappers.foreach(w ⇒ { + val mdl = w.proxy val mdlName = mdl.getName val mdlId = mdl.getId val mdlVer = mdl.getVersion @@ -577,7 +578,7 @@ object NCDeployManager extends NCService with DecorateAsScala { throw new NCE(s"Model element ID '${elm.getId}' does not match '$ID_REGEX' regex in: $mdlId") }) - if (U.containsDups(models.map(_.getId).toList)) + if (U.containsDups(wrappers.map(_.proxy.getId).toList)) throw new NCE("Duplicate model IDs detected.") super.start() @@ -588,8 +589,8 @@ object NCDeployManager extends NCService with DecorateAsScala { if (modelFactory != null) modelFactory.terminate() - if (models != null) - models.clear() + if (wrappers != null) + wrappers.clear() super.stop() } @@ -598,7 +599,7 @@ object NCDeployManager extends NCService with DecorateAsScala { * * @return */ - def getModels: Seq[NCModelWrapper] = models + def getModels: Seq[NCModelWrapper] = wrappers /** * Permutes and drops duplicated. diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala index 52b0767..69c0ba5 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala @@ -49,8 +49,8 @@ trait NCProbeInspection extends NCInspectionService { val warns = mutable.Buffer.empty[String] val suggs = mutable.Buffer.empty[String] - NCModelManager.getModel(mdlId) match { - case Some(x) ⇒ body(x, args, suggs, warns, errs) + NCModelManager.getModelWrapper(mdlId) match { + case Some(x) ⇒ body(x.proxy, args, suggs, warns, errs) case None ⇒ errs += s"Model not found: $mdlId" } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala index 40760e1..446c2e4 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala @@ -23,9 +23,8 @@ import io.opencensus.trace.Span import org.apache.nlpcraft.common._ import org.apache.nlpcraft.common.ascii.NCAsciiTable import org.apache.nlpcraft.model._ -import org.apache.nlpcraft.model.impl.NCModelWrapper import org.apache.nlpcraft.model.intent.impl.NCIntentScanner -import org.apache.nlpcraft.probe.mgrs.deploy._ +import org.apache.nlpcraft.probe.mgrs.deploy.{NCModelWrapper, _} import scala.collection.JavaConverters._ import scala.collection.convert.DecorateAsScala @@ -36,7 +35,7 @@ import scala.util.control.Exception._ */ object NCModelManager extends NCService with DecorateAsScala { // Deployed models keyed by their IDs. - @volatile private var models: Map[String, NCModelWrapper] = _ + @volatile private var wrappers: Map[String, NCModelWrapper] = _ // Access mutex. private final val mux = new Object() @@ -47,30 +46,32 @@ object NCModelManager extends NCService with DecorateAsScala { val tbl = NCAsciiTable("Model ID", "Name", "Ver.", "Elements", "Synonyms") mux.synchronized { - models = NCDeployManager.getModels.map(mdl ⇒ { - mdl.onInit() + wrappers = NCDeployManager.getModels.map(w ⇒ { + w.proxy.onInit() - mdl.proxy.getId → mdl + w.proxy.getId → w }).toMap - models.values.foreach(mdl ⇒ { - val synCnt = mdl.syns.values.flatMap(_.values).flatten.size + wrappers.values.foreach(w ⇒ { + val mdl = w.proxy + + val synCnt = w.synonyms.values.flatMap(_.values).flatten.size tbl += ( mdl.getId, mdl.getName, mdl.getVersion, - mdl.elms.keySet.size, + w.elements.keySet.size, synCnt ) }) } - tbl.info(logger, Some(s"Models deployed: ${models.size}\n")) + tbl.info(logger, Some(s"Models deployed: ${wrappers.size}\n")) addTags( span, - "deployedModels" → models.values.map(_.getId).mkString(",") + "deployedModels" → wrappers.values.map(_.proxy.getId).mkString(",") ) super.start() @@ -96,8 +97,8 @@ object NCModelManager extends NCService with DecorateAsScala { */ override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒ mux.synchronized { - if (models != null) - models.values.foreach(m ⇒ discardModel(m)) + if (wrappers != null) + wrappers.values.foreach(m ⇒ discardModel(m.proxy)) } super.stop() @@ -108,10 +109,10 @@ object NCModelManager extends NCService with DecorateAsScala { * * @return */ - def getAllModels(parent: Span = null): List[NCModelWrapper] = + def getAllModelWrappers(parent: Span = null): List[NCModelWrapper] = startScopedSpan("getAllModels", parent) { _ ⇒ mux.synchronized { - models.values.toList + wrappers.values.toList } } @@ -120,10 +121,10 @@ object NCModelManager extends NCService with DecorateAsScala { * @param mdlId Model ID. * @return */ - def getModel(mdlId: String, parent: Span = null): Option[NCModelWrapper] = + def getModelWrapper(mdlId: String, parent: Span = null): Option[NCModelWrapper] = startScopedSpan("getModel", parent, "modelId" → mdlId) { _ ⇒ mux.synchronized { - models.get(mdlId) + wrappers.get(mdlId) } } @@ -137,13 +138,14 @@ object NCModelManager extends NCService with DecorateAsScala { */ def getModelInfo(mdlId: String, parent: Span = null): java.util.Map[String, Any] = startScopedSpan("getModel", parent, "mdlId" → mdlId) { _ ⇒ - val mdl = mux.synchronized { models.get(mdlId) }.getOrElse(throw new NCE(s"Model not found: '$mdlId'")) + val w = mux.synchronized { wrappers.get(mdlId) }.getOrElse(throw new NCE(s"Model not found: '$mdlId'")) + val mdl = w.proxy val data = new util.HashMap[String, Any]() data.put("macros", mdl.getMacros) data.put("synonyms", mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms).toMap.asJava) - data.put("samples", NCIntentScanner.scanIntentsSamples(mdl.proxy).samples.map(p ⇒ p._1 → p._2.asJava).asJava) + data.put("samples", NCIntentScanner.scanIntentsSamples(mdl).samples.map(p ⇒ p._1 → p._2.asJava).asJava) data } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala index cfba3e7..ede7298 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala @@ -23,7 +23,7 @@ import com.typesafe.scalalogging.LazyLogging import io.opencensus.trace.Span import org.apache.nlpcraft.common.nlp._ import org.apache.nlpcraft.common.{NCService, _} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import scala.collection.Map import scala.language.implicitConversions diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala index 850b7a8..8acbb5a 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala @@ -315,16 +315,14 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { logger.info(s"REJECT response $msgName sent [srvReqId=$srvReqId, response=${errMsg.get}]") } - val mdl = NCModelManager - .getModel(mdlId, span) - .getOrElse(throw new NCE(s"Model not found: $mdlId")) + val w = NCModelManager.getModelWrapper(mdlId, span).getOrElse(throw new NCE(s"Model not found: $mdlId")) var errData: Option[(String, Int)] = None val validNlpSens = nlpSens.flatMap(nlpSen ⇒ try { - NCValidateManager.preValidate(mdl, nlpSen, span) + NCValidateManager.preValidate(w, nlpSen, span) Some(nlpSen) } @@ -361,14 +359,14 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { val sensSeq = validNlpSens.flatMap(nlpSen ⇒ { // Independent of references. - NCDictionaryEnricher.enrich(mdl, nlpSen, senMeta, span) - NCSuspiciousNounsEnricher.enrich(mdl, nlpSen, senMeta, span) - NCStopWordEnricher.enrich(mdl, nlpSen, senMeta, span) + NCDictionaryEnricher.enrich(w, nlpSen, senMeta, span) + NCSuspiciousNounsEnricher.enrich(w, nlpSen, senMeta, span) + NCStopWordEnricher.enrich(w, nlpSen, senMeta, span) case class Holder(enricher: NCProbeEnricher, getNotes: () ⇒ Seq[NCNlpSentenceNote]) def get(name: String, e: NCProbeEnricher): Option[Holder] = - if (mdl.getEnabledBuiltInTokens.contains(name)) + if (w.proxy.getEnabledBuiltInTokens.contains(name)) Some(Holder(e, () ⇒ nlpSen.flatten.filter(_.noteType == name))) else None @@ -394,7 +392,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { def get(): Seq[NCNlpSentenceNote] = h.getNotes().sortBy(p ⇒ (p.tokenIndexes.head, p.noteType)) val notes1 = get() - h → h.enricher.enrich(mdl, nlpSen, senMeta, span) + h → h.enricher.enrich(w, nlpSen, senMeta, span) val notes2 = get() @@ -434,7 +432,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { }).toMap // Loop has sense if model is complex (has user defined parsers or DSL based synonyms) - continue = NCModelEnricher.isComplex(mdl) && res.exists { case (_, same) ⇒ !same } + continue = NCModelEnricher.isComplex(w) && res.exists { case (_, same) ⇒ !same } if (DEEP_DEBUG) if (continue) { @@ -464,7 +462,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { // Final validation before execution. try - sensSeq.foreach(NCValidateManager.postValidate(mdl, _, span)) + sensSeq.foreach(NCValidateManager.postValidate(w, _, span)) catch { case e: NCValidateException ⇒ val (errMsg, errCode) = getError(e.code) @@ -487,13 +485,13 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { val meta = mutable.HashMap.empty[String, Any] ++ senMeta val req = NCRequestImpl(meta, srvReqId) - var senVars = mdl.makeVariants(srvReqId, sensSeq) + var senVars = w.makeVariants(srvReqId, sensSeq) // Sentence variants can be filtered by model. val fltSenVars: Seq[(NCVariant, Int)] = senVars. zipWithIndex. - flatMap { case (variant, i) ⇒ if (mdl.onParsedVariant(variant)) Some(variant, i) else None } + flatMap { case (variant, i) ⇒ if (w.proxy.onParsedVariant(variant)) Some(variant, i) else None } senVars = fltSenVars.map(_._1) val allVars = senVars.flatMap(_.asScala) @@ -528,7 +526,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { // Create model query context. val ctx: NCContext = new NCContext { override lazy val getRequest: NCRequest = req - override lazy val getModel: NCModel = mdl + override lazy val getModel: NCModel = w.proxy override lazy val getServerRequestId: String = srvReqId override lazy val getConversation: NCConversation = new NCConversation { @@ -546,7 +544,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { logKey = U.mkLogHolderKey(srvReqId) - val meta = mdl.getMetadata + val meta = w.proxy.getMetadata meta.synchronized { meta.put(logKey, logHldr) @@ -572,7 +570,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { def onFinish(): Unit = { if (logKey != null) - mdl.getMetadata.remove(logKey) + w.proxy.getMetadata.remove(logKey) span.end() } @@ -582,16 +580,16 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { // Execute model query asynchronously. U.asFuture( _ ⇒ { - var res = mdl.onContext(ctx) + var res = w.proxy.onContext(ctx) start = System.currentTimeMillis() - if (res == null && mdl.solver != null) + if (res == null && w.solver != null) startScopedSpan("intentMatching", span) { _ ⇒ - res = mdl.solver.solve(solverIn, span) + res = w.solver.solve(solverIn, span) } - if (res == null && mdl.solver == null) + if (res == null && w.solver == null) throw new IllegalStateException("No intents and no results from model callbacks.") recordStats(M_USER_LATENCY_MS → (System.currentTimeMillis() - start)) @@ -625,7 +623,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { if (e.getCause != null) logger.info(s"Rejection cause:", e.getCause) - val res = mdl.onRejection(solverIn.intentMatch, e) + val res = w.proxy.onRejection(solverIn.intentMatch, e) if (res != null) respondWithResult(res, None) @@ -654,7 +652,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { logger.error(s"Unexpected error for server request ID: $srvReqId", e) - val res = mdl.onError(ctx, e) + val res = w.proxy.onError(ctx, e) if (res != null) respondWithResult(res, None) @@ -680,7 +678,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats { "resBody" → res.getBody ) - val res0 = mdl.onResult(solverIn.intentMatch, res) + val res0 = w.proxy.onResult(solverIn.intentMatch, res) respondWithResult(if (res0 != null) res0 else res, if (logHldr != null) Some(logHldr.toJson) else None) } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala index 4905273..bf49bf7 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/dictionary/NCDictionaryEnricher.scala @@ -24,7 +24,7 @@ import org.apache.nlpcraft.common.nlp._ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.nlp.dict._ import org.apache.nlpcraft.common.{NCService, _} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.collection.Map @@ -57,7 +57,7 @@ object NCDictionaryEnricher extends NCProbeEnricher { override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ ns.foreach(t ⇒ { // Dictionary. diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala index 4286b34..5c3d71e 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala @@ -25,7 +25,7 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.nlp.numeric.{NCNumeric, NCNumericManager} import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSentenceToken} import org.apache.nlpcraft.common.{NCE, NCService} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.collection.JavaConverters._ @@ -239,7 +239,7 @@ object NCLimitEnricher extends NCProbeEnricher { override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ val notes = mutable.HashSet.empty[NCNlpSentenceNote] val numsMap = NCNumericManager.find(ns).filter(_.unit.isEmpty).map(p ⇒ p.tokens → p).toMap diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala index c228c97..4551bc0 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala @@ -24,10 +24,10 @@ import io.opencensus.trace.Span import org.apache.nlpcraft.common._ import org.apache.nlpcraft.common.nlp.{NCNlpSentenceToken, _} import org.apache.nlpcraft.model._ -import org.apache.nlpcraft.model.impl.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCRequestImpl import org.apache.nlpcraft.probe.mgrs.NCSynonym +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import scala.collection.JavaConverters._ import scala.collection.convert.DecorateAsScala @@ -298,15 +298,15 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { */ private def alreadyMarked(toks: Seq[NCNlpSentenceToken], elemId: String): Boolean = toks.forall(_.isTypeOf(elemId)) - def isComplex(mdl: NCModelWrapper): Boolean = mdl.synsDsl.nonEmpty || !mdl.getParsers.isEmpty + def isComplex(mdl: NCModelWrapper): Boolean = mdl.synonymsDsl.nonEmpty || !mdl.proxy.getParsers.isEmpty @throws[NCE] - override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = + override def enrich(w: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → w.proxy.getId, "txt" → ns.text) { span ⇒ - val jiggleFactor = mdl.getJiggleFactor + val jiggleFactor = w.proxy.getJiggleFactor val cache = mutable.HashSet.empty[Seq[Int]] val matches = ArrayBuffer.empty[ElementMatch] @@ -353,7 +353,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { var seq: Seq[Seq[Complex]] = null // Attempt to match each element. - for (elm ← mdl.elms.values if !alreadyMarked(toks, elm.getId)) { + for (elm ← w.elements.values if !alreadyMarked(toks, elm.getId)) { var found = false def addMatch( @@ -366,21 +366,21 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { } // Optimization - plain synonyms can be used only on first iteration - if (mdl.syns.nonEmpty && !ns.exists(_.isUser)) - for (syn ← fastAccess(mdl.syns, elm.getId, toks.length) if !found) + if (w.synonyms.nonEmpty && !ns.exists(_.isUser)) + for (syn ← fastAccess(w.synonyms, elm.getId, toks.length) if !found) if (syn.isMatch(toks)) addMatch(elm, toks, syn, Seq.empty) - if (mdl.synsDsl.nonEmpty) { + if (w.synonymsDsl.nonEmpty) { found = false if (collapsedSens == null) - collapsedSens = mdl.makeVariants(ns.srvReqId, ns.clone().collapse()).map(_.asScala) + collapsedSens = w.makeVariants(ns.srvReqId, ns.clone().collapse()).map(_.asScala) if (seq == null) seq = convert(ns, collapsedSens, toks) - for (comb ← seq; syn ← fastAccess(mdl.synsDsl, elm.getId, comb.length) if !found) + for (comb ← seq; syn ← fastAccess(w.synonymsDsl, elm.getId, comb.length) if !found) if (syn.isMatch(comb.map(_.data))) addMatch(elm, toks, syn, comb.filter(_.isToken).map(_.token)) } @@ -393,7 +393,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { startScopedSpan("jiggleProc", span, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → w.proxy.getId, "txt" → ns.text) { _ ⇒ // Iterate over depth-limited permutations of the original sentence with and without stopwords. jiggle(ns, jiggleFactor).foreach(procPerm) @@ -414,30 +414,30 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { for ((m, idx) ← matches.zipWithIndex) { if (DEEP_DEBUG) logger.trace( - s"Model '${mdl.getId}' element found (${idx + 1} of $matchCnt) [" + + s"Model '${w.proxy.getId}' element found (${idx + 1} of $matchCnt) [" + s"elementId=${m.element.getId}, " + s"synonym=${m.synonym}, " + s"tokens=${tokString(m.tokens)}" + s"]" ) - + val elm = m.element val syn = m.synonym - + val tokIdxs = m.tokens.map(_.index) val direct = syn.isDirect && (tokIdxs == tokIdxs.sorted) mark(ns, elem = elm, toks = m.tokens, direct = direct, syn = Some(syn), metaOpt = None, parts = m.parts) } - val parsers = mdl.getParsers + val parsers = w.proxy.getParsers for (parser ← parsers.asScala) { parser.onInit() startScopedSpan("customParser", span, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → w.proxy.getId, "txt" → ns.text) { _ ⇒ def to(t: NCNlpSentenceToken): NCCustomWord = new NCCustomWord { @@ -459,7 +459,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { val res = parser.parse( NCRequestImpl(senMeta, ns.srvReqId), - mdl, + w.proxy, ns.map(to).asJava, ns.flatten.distinct.filter(!_.isNlp).map(n ⇒ { val noteId = n.noteType @@ -495,7 +495,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala { if (!alreadyMarked(matchedToks, elemId)) mark( ns, - elem = mdl.elms.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")), + elem = w.elements.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")), toks = matchedToks, direct = true, syn = None, diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala index d223a01..4bd2b03 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala @@ -24,7 +24,7 @@ import org.apache.nlpcraft.common.makro.NCMacroParser import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSentenceToken} import org.apache.nlpcraft.common.{NCE, NCService} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.collection.JavaConverters._ @@ -141,7 +141,7 @@ object NCRelationEnricher extends NCProbeEnricher { override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ // Tries to grab tokens direct way. // Example: A, B, C ⇒ ABC, AB, BC .. (AB will be processed first) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala index 67e4ec5..e4cb56f 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala @@ -24,7 +24,7 @@ import org.apache.nlpcraft.common.NCService import org.apache.nlpcraft.common.makro.NCMacroParser import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSentenceToken} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.collection.JavaConverters._ @@ -418,7 +418,7 @@ object NCSortEnricher extends NCProbeEnricher { override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, meta: Map[String, Serializable], parent: Span): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ val notes = mutable.HashSet.empty[NCNlpSentenceNote] diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala index 8d52564..e293e73 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/stopword/NCStopWordEnricher.scala @@ -23,7 +23,7 @@ import io.opencensus.trace.Span import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceToken} import org.apache.nlpcraft.common.{NCE, NCService, U} -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.annotation.tailrec @@ -210,7 +210,7 @@ object NCStopWordEnricher extends NCProbeEnricher { def mark(stems: Set[String], f: Boolean): Unit = ns.filter(t ⇒ stems.contains(t.stem)).foreach(t ⇒ ns.fixNote(t.getNlpNote, "stopWord" → f)) - startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, "modelId" → mdl.getId, "txt" → ns.text) { _ ⇒ + startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ mark(mdl.exclStopWordsStems, f = false) mark(mdl.addStopWordsStems, f = true) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala index e797051..5d234db 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/suspicious/NCSuspiciousNounsEnricher.scala @@ -22,7 +22,7 @@ import java.io.Serializable import io.opencensus.trace.Span import org.apache.nlpcraft.common.{NCE, NCService} import org.apache.nlpcraft.common.nlp._ -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher import scala.collection.Map @@ -43,7 +43,7 @@ object NCSuspiciousNounsEnricher extends NCProbeEnricher { override def enrich(mdl: NCModelWrapper, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit = startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, - "modelId" → mdl.getId, + "modelId" → mdl.proxy.getId, "txt" → ns.text) { _ ⇒ ns.filter(t ⇒ mdl.suspWordsStems.contains(t.stem)).foreach(t ⇒ ns.fixNote(t.getNlpNote, "suspNoun" → true)) } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala index 6cde756..0412247 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/validate/NCValidateManager.scala @@ -22,7 +22,7 @@ import io.opencensus.trace.Span import org.apache.tika.langdetect.OptimaizeLangDetector import org.apache.nlpcraft.common.NCService import org.apache.nlpcraft.common.nlp.NCNlpSentence -import org.apache.nlpcraft.model.impl.NCModelWrapper +import org.apache.nlpcraft.probe.mgrs.deploy.NCModelWrapper /** * Probe pre/post enrichment validator. @@ -46,16 +46,17 @@ object NCValidateManager extends NCService with LazyLogging { /** * - * @param mdl Model decorator. + * @param w Model decorator. * @param ns Sentence to validate. * @param parent Parent tracing span. */ @throws[NCValidateException] - def preValidate(mdl: NCModelWrapper, ns: NCNlpSentence, parent: Span = null): Unit = + def preValidate(w: NCModelWrapper, ns: NCNlpSentence, parent: Span = null): Unit = startScopedSpan("validate", parent, "srvReqId" → ns.srvReqId, "txt" → ns.text, - "modelId" → mdl.getId) { _ ⇒ + "modelId" → w.proxy.getId) { _ ⇒ + val mdl = w.proxy if (!mdl.isNotLatinCharsetAllowed && !ns.text.matches("""[\s\w\p{Punct}]+""")) throw NCValidateException("ALLOW_NON_LATIN_CHARSET") @@ -71,16 +72,17 @@ object NCValidateManager extends NCService with LazyLogging { /** * - * @param mdl Model decorator. + * @param w Model decorator. * @param ns Sentence to validate. * @param parent Optional parent span. */ @throws[NCValidateException] - def postValidate(mdl: NCModelWrapper, ns: NCNlpSentence, parent: Span = null): Unit = + def postValidate(w: NCModelWrapper, ns: NCNlpSentence, parent: Span = null): Unit = startScopedSpan("validate", parent, "srvReqId" → ns.srvReqId, "txt" → ns.text, - "modelId" → mdl.getId) { _ ⇒ + "modelId" → w.proxy.getId) { _ ⇒ + val mdl = w.proxy val types = ns.flatten.filter(!_.isNlp).map(_.noteType).distinct val overlapNotes = ns.map(tkn ⇒ types.flatMap(tp ⇒ tkn.getNotes(tp))).filter(_.size > 1).flatten
