This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41-1
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41-1 by this push:
new bdc8a1b WIP.
bdc8a1b is described below
commit bdc8a1bfc12b93921b60e704ede8967248774c6b
Author: Sergey Kamov <[email protected]>
AuthorDate: Thu Sep 10 11:45:51 2020 +0300
WIP.
---
nlpcraft/src/main/resources/nlpcraft.conf | 3 +
.../nlpcraft/common/inspections/NCInspection.scala | 56 -------------
.../common/inspections/NCInspectionParameter.scala | 53 ------------
.../common/inspections/NCInspectionResult.scala | 69 ----------------
.../common/inspections/NCInspectionService.scala | 60 --------------
.../common/inspections/impl/NCInspectionImpl.scala | 47 -----------
.../impl/NCInspectionParameterImpl.scala | 28 -------
.../inspections/impl/NCInspectionResultImpl.scala | 56 -------------
.../test/impl/NCTestAutoModelValidatorImpl.scala | 10 +--
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 3 -
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 40 ++++------
.../probe/mgrs/deploy/NCDeployManager.scala | 93 +++++++++++++++-------
.../mgrs/deploy}/NCIntentScanner.scala | 54 ++++++++++---
.../probe/mgrs/deploy/NCModelWrapper.scala | 5 +-
.../mgrs/inspections/NCInspectionManager.scala | 70 ----------------
.../inspectors/NCIntentsInspection.scala | 70 ----------------
.../inspectors/NCMacrosInspection.scala | 47 -----------
.../inspections/inspectors/NCProbeInspection.scala | 86 --------------------
.../inspectors/NCSynonymsInspection.scala | 72 -----------------
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 28 -------
.../nlpcraft/server/probe/NCProbeManager.scala | 54 +++++--------
.../server/sugsyn/NCSuggestSynonymManager.scala | 4 +-
22 files changed, 155 insertions(+), 853 deletions(-)
diff --git a/nlpcraft/src/main/resources/nlpcraft.conf
b/nlpcraft/src/main/resources/nlpcraft.conf
index ad65695..7ef1e8f 100644
--- a/nlpcraft/src/main/resources/nlpcraft.conf
+++ b/nlpcraft/src/main/resources/nlpcraft.conf
@@ -309,6 +309,9 @@ nlpcraft {
# Maximum execution result size in bytes. Default value is 1M.
# When exceeded the request will be automatically rejected.
resultMaxSizeBytes = 1048576
+
+ // TODO:
+ synonymsWarnValue = 10000
}
# Basic NLP toolkit to use on both server and probes. Possible values:
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
deleted file mode 100644
index 7a0a855..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections
-
-/**
- * Inspection descriptor.
- */
-trait NCInspection {
- /**
- * Globally unique, user-visible name of the inspection.
- *
- * @return
- */
- def name(): String
-
- /**
- * Short, one-line, description.
- *
- * @return
- */
- def synopsis(): String
-
- /**
- *
- * @return
- */
- def parameters(): java.util.List[NCInspectionParameter]
-
- /**
- * Full description of this inspection additionally to the synopsis.
- *
- * @return
- */
- def description(): String
-
- /**
- *
- * @return
- */
- def isServerSide: Boolean
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
deleted file mode 100644
index 25378e7..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections
-
-/**
- * Parameter descriptor for the inspection.
- */
-trait NCInspectionParameter {
- /**
- *
- * @return
- */
- def name(): String
-
- /**
- *
- * @return
- */
- def value(): String
-
- /**
- *
- * @return
- */
- def valueType(): String
-
- /**
- *
- * @return
- */
- def synopsis(): String
-
- /**
- *
- * @return
- */
- def description(): String
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
deleted file mode 100644
index 5118a0c..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections
-
-/**
- *
- */
-trait NCInspectionResult {
- /**
- *
- */
- def inspectionId(): String
-
- /**
- *
- */
- def modelId(): String
-
- /**
- *
- * @return
- */
- def inspectionArguments(): String
-
- /**
- *
- * @return
- */
- def durationMs(): Long
-
- /**
- *
- * @return
- */
- def timestamp(): Long
-
- /**
- *
- * @return
- */
- def errors(): java.util.List[String]
-
- /**
- *
- * @return
- */
- def warnings(): java.util.List[String]
-
- /**
- *
- * @return
- */
- def suggestions(): java.util.List[AnyRef]
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
deleted file mode 100644
index 0222e20..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-package org.apache.nlpcraft.common.inspections
-
-import java.util.concurrent.{ExecutorService, Executors}
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.util.NCUtils
-
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
-
-/**
- * Base trait for inspection implementation.
- */
-trait NCInspectionService extends NCService {
- /** */
- @volatile private var pool: ExecutorService = _
-
- /** */
- @volatile private var executor: ExecutionContextExecutor = _
-
- /**
- *
- * @param mdlId
- * @param inspName
- * @param args
- * @param parent
- * @return
- */
- def inspect(mdlId: String, inspName: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
-
- /**
- *
- * @return
- */
- def getExecutor = executor
-
- /**
- *
- * @return
- */
- def getName: String
-
- override def start(parent: Span): NCService =
- startScopedSpan("start", parent) { _ ⇒
- pool = Executors.newCachedThreadPool()
- executor = ExecutionContext.fromExecutor(pool)
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit =
- startScopedSpan("stop", parent) { _ ⇒
- super.stop(parent)
-
- NCUtils.shutdownPools(pool)
-
- pool = null
- executor = null
- }
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
deleted file mode 100644
index 11a8b24..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections.impl
-
-import org.apache.nlpcraft.common.inspections.{NCInspection,
NCInspectionParameter}
-
-import scala.collection.JavaConverters.seqAsJavaListConverter
-
-case class NCInspectionImpl(
- name: String,
- synopsis: String,
- parameters: java.util.List[NCInspectionParameter],
- description: String,
- isServerSide: Boolean
-) extends NCInspection
-
-object NCInspectionImpl {
- def apply(
- name: String,
- synopsis: String,
- parameters: Seq[NCInspectionParameter] = Seq.empty,
- description: String,
- isServerSide: Boolean,
- ) : NCInspectionImpl =
- NCInspectionImpl(
- name,
- synopsis,
- parameters = parameters.asJava,
- description,
- isServerSide
- )
-}
\ No newline at end of file
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
deleted file mode 100644
index 9c02d73..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections.impl
-
-import org.apache.nlpcraft.common.inspections.NCInspectionParameter
-
-case class NCInspectionParameterImpl(
- name: String,
- value: String,
- valueType: String,
- synopsis: String,
- description: String
-) extends NCInspectionParameter
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
deleted file mode 100644
index ac70546..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections.impl
-
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-
-import scala.collection.JavaConverters.seqAsJavaListConverter
-
-case class NCInspectionResultImpl(
- inspectionId: String,
- modelId: String,
- inspectionArguments: String ,
- durationMs: Long,
- timestamp: Long,
- errors: java.util.List[String],
- warnings: java.util.List[String],
- suggestions: java.util.List[AnyRef]
-) extends NCInspectionResult
-
-object NCInspectionResultImpl {
- def apply(
- inspectionId: String,
- modelId: String,
- inspectionArguments: Option[String] = None,
- durationMs: Long,
- timestamp: Long,
- errors: Seq[String] = Seq.empty,
- warnings: Seq[String] = Seq.empty,
- suggestions: Seq[AnyRef] = Seq.empty
- ): NCInspectionResultImpl =
- new NCInspectionResultImpl(
- inspectionId,
- modelId,
- inspectionArguments.orNull,
- durationMs,
- timestamp,
- errors.asJava,
- warnings.asJava,
- suggestions.asJava
- )
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
index 3230490..9cc7a80 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
@@ -20,9 +20,9 @@ package org.apache.nlpcraft.model.tools.test.impl
import com.typesafe.scalalogging.LazyLogging
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.util.NCUtils
-import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.model.tools.embedded.NCEmbeddedProbe
import org.apache.nlpcraft.model.tools.test.NCTestClientBuilder
+import org.apache.nlpcraft.probe.mgrs.deploy.NCIntentScanner
import org.apache.nlpcraft.model._
/**
@@ -61,13 +61,7 @@ private [test] object NCTestAutoModelValidatorImpl extends
LazyLogging {
val samples =
classes.
map(_.getDeclaredConstructor().newInstance()).
- map(mdl ⇒ {
- val res = NCIntentScanner.scanIntentsSamples(mdl)
-
- res.warnings.foreach(w ⇒ logger.warn(w))
-
- mdl.getId → res.samples.toMap
- }).
+ map(mdl ⇒ mdl.getId → NCIntentScanner.scanSamples(mdl).toMap).
toMap.
filter(_._2.nonEmpty)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index dad67ab..b583100 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -36,7 +36,6 @@ import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
import org.apache.nlpcraft.probe.mgrs.deploy.NCDeployManager
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
-import org.apache.nlpcraft.probe.mgrs.inspections.NCInspectionManager
import org.apache.nlpcraft.probe.mgrs.lifecycle.NCLifecycleManager
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
@@ -423,7 +422,6 @@ private [probe] object NCProbeBoot extends LazyLogging with
NCOpenCensusTrace {
NCNlpCoreManager.start(span)
NCNumericManager.start(span)
NCDeployManager.start(span)
- NCInspectionManager.start(span)
NCModelManager.start(span)
NCCommandManager.start(span)
NCDictionaryManager.start(span)
@@ -463,7 +461,6 @@ private [probe] object NCProbeBoot extends LazyLogging with
NCOpenCensusTrace {
NCDictionaryManager.stop(span)
NCCommandManager.stop(span)
NCModelManager.stop(span)
- NCInspectionManager.stop(span)
NCDeployManager.stop(span)
NCNumericManager.stop(span)
NCNlpCoreManager.stop(span)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 6abfc27..5442d71 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -22,21 +22,19 @@ import java.util.concurrent.{ExecutorService, Executors}
import com.google.gson.Gson
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.util.NCUtils
+import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.model.NCToken
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
-import org.apache.nlpcraft.probe.mgrs.inspections.NCInspectionManager
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
import scala.collection.JavaConverters._
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor}
-import scala.util.{Failure, Success}
/**
* Probe commands processor.
@@ -107,34 +105,28 @@ object NCCommandManager extends NCService {
span
)
- case "S2P_PROBE_INSPECTION" ⇒
- NCInspectionManager.inspect(
- mdlId = msg.data[String]("mdlId"),
- inspName = msg.data[String]("inspName"),
- args = msg.dataOpt[String]("args"),
- span
- ).onComplete {
- case Success(res) ⇒
- NCConnectionManager.send(
- NCProbeMessage(
- "P2S_PROBE_INSPECTION",
- "reqGuid" → msg.getGuid,
- "resp" → GSON.toJson(res)
- ),
- span
- )
+ case "S2P_MODEL_INFO" ⇒
+ val mdlId = msg.data[String]("mdlId")
- case Failure(e) ⇒ logger.error(s"Message cannot be
processed: $msg", e)
- }(executor)
+ val w =
NCModelManager.getModelWrapper(mdlId).getOrElse(throw new NCE(s"Model not
found: '$mdlId'"))
- case "S2P_MODEL_INFO" ⇒
- val res =
NCModelManager.getModelInfo(msg.data[String]("mdlId"))
+ val macros =
w.proxy.getMacros.asInstanceOf[Serializable]
+ val syns = w.proxy.getElements.asScala.
+ map(p ⇒ p.getId →
p.getSynonyms).toMap.asJava.asInstanceOf[Serializable]
+ val samples = w.samples.map(p ⇒ p._1 → p._2.asJava).
+ asJava.asInstanceOf[Serializable]
NCConnectionManager.send(
NCProbeMessage(
"P2S_MODEL_INFO",
"reqGuid" → msg.getGuid,
- "resp" → GSON.toJson(res)
+ "resp" → GSON.toJson(
+ Map(
+ "macros" → macros,
+ "synonyms" → syns,
+ "samples" → samples
+ ).asJava
+ )
),
span
)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index 571de80..f4faa8a 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -29,7 +29,7 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.util.NCUtils.{DSL_FIX, REGEX_FIX}
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.model.factories.basic.NCBasicModelFactory
-import org.apache.nlpcraft.model.intent.impl.{NCIntentScanner, NCIntentSolver}
+import org.apache.nlpcraft.model.intent.impl.NCIntentSolver
import org.apache.nlpcraft.probe.mgrs.NCSynonymChunkKind.{DSL, REGEX, TEXT}
import org.apache.nlpcraft.probe.mgrs.{NCSynonym, NCSynonymChunk, deploy}
import org.apache.nlpcraft.probe.mgrs.model.NCModelSynonymDslCompiler
@@ -59,6 +59,8 @@ object NCDeployManager extends NCService with DecorateAsScala
{
def modelFactoryProps: Option[Map[String, String]] =
getMapOpt(s"$pre.modelFactory.properties")
def models: Seq[String] = getStringList(s"$pre.models")
def jarsFolder: Option[String] = getStringOpt(s"$pre.jarsFolder")
+ // TODO: property name.
+ def synonymsWarnValue: Int = getInt(s"$pre.synonymsWarnValue")
}
/**
@@ -93,6 +95,13 @@ object NCDeployManager extends NCService with
DecorateAsScala {
*/
@throws[NCE]
private def wrap(mdl: NCModel): NCModelWrapper = {
+ val mdlId = mdl.getId
+
+ @throws[NCE]
+ def checkCollection(name: String, col: Any): Unit =
+ if (col == null)
+ throw new NCE(s"Collection can be empty but cannot be null
[modelId=$mdlId, name=$name]")
+
checkCollection("additionalStopWords", mdl.getAdditionalStopWords)
checkCollection("elements", mdl.getElements)
checkCollection("enabledBuiltInTokens", mdl.getEnabledBuiltInTokens)
@@ -102,16 +111,23 @@ object NCDeployManager extends NCService with
DecorateAsScala {
checkCollection("macros", mdl.getMacros)
checkCollection("metadata", mdl.getMetadata)
+ val allSyns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
+
+ mdl.getMacros.asScala.keys.foreach(makro ⇒
+ if (!allSyns.exists(_.contains(makro)))
+ logger.warn(s"Unused macro [modelId=$mdlId, makro=$makro]")
+ )
+
// Scan for intent annotations in the model class.
val intents = NCIntentScanner.scan(mdl)
- val mdlId = mdl.getId
-
val parser = new NCMacroParser
// Initialize macro parser.
mdl.getMacros.asScala.foreach(t ⇒ parser.addMacro(t._1, t._2))
+ checkSynonyms(mdl, parser)
+
var solver: NCIntentSolver = null
if (intents.nonEmpty) {
@@ -169,19 +185,19 @@ object NCDeployManager extends NCService with
DecorateAsScala {
if (cnt > maxCnt)
throw new NCE(s"Too many synonyms detected [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"max=$maxCnt" +
s"]")
if (value == null)
logger.trace(s"Synonym #${syns.size} added [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"synonym=${chunks.mkString(" ")}" +
s"]")
else
logger.trace(s"Synonym #${syns.size} added [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"synonym=${chunks.mkString(" ")}, " +
s"value=$value" +
@@ -190,7 +206,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
else
logger.trace(
s"Synonym already added (ignoring) [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"synonym=${chunks.mkString(" ")}, " +
s"value=$value" +
@@ -229,7 +245,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
if (U.containsDups(synsChunks.flatten))
logger.trace(s"Element synonyms duplicate (ignoring) [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"synonym=${synsChunks.diff(synsChunks.distinct).distinct.map(_.mkString(",")).mkString(";")}"
+
s"]"
@@ -246,7 +262,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
if (U.containsDups(valNames))
logger.trace(s"Element values names duplicate (ignoring) [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"names=${valNames.diff(valNames.distinct).distinct.mkString(",")}" +
s"]"
@@ -279,7 +295,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
if (U.containsDups(chunks.toList))
logger.trace(s"Element synonyms duplicate (ignoring) [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"elementId=$elmId, " +
s"value=$valId, " +
s"synonym=${chunks.diff(chunks.distinct).distinct.map(_.mkString(",")).mkString(";")}"
+
@@ -317,9 +333,9 @@ object NCDeployManager extends NCService with
DecorateAsScala {
// Check for DSl alias uniqueness.
if (U.containsDups(allAliases)) {
for (dupAlias ← allAliases.diff(allAliases.distinct))
- logger.warn(s"Duplicate DSL alias '$dupAlias' found for model:
${mdl.getId}")
+ logger.warn(s"Duplicate DSL alias '$dupAlias' found for model:
$mdlId")
- throw new NCE(s"Duplicate DSL aliases found for model
'${mdl.getId}'- check log messages.")
+ throw new NCE(s"Duplicate DSL aliases found for model '$mdlId'-
check log messages.")
}
val idAliasDups =
@@ -331,9 +347,9 @@ object NCDeployManager extends NCService with
DecorateAsScala {
// Check that DSL aliases don't intersect with element IDs.
if (idAliasDups.nonEmpty) {
for (dup ← idAliasDups)
- logger.warn(s"Duplicate element ID and DSL alias '$dup' found
for model: ${mdl.getId}")
+ logger.warn(s"Duplicate element ID and DSL alias '$dup' found
for model: $mdlId")
- throw new NCE(s"Duplicate element ID and DSL aliases found for
model '${mdl.getId}'- check log messages.")
+ throw new NCE(s"Duplicate element ID and DSL aliases found for
model '$mdlId'- check log messages.")
}
// Check for synonym dups across all elements.
@@ -342,7 +358,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
syns.groupBy(p ⇒ (p.synonym.mkString(" "),
p.synonym.isDirect)) if holders.size > 1 && isDirect
) {
logger.trace(s"Duplicate synonym detected (ignoring) [" +
- s"model=${mdl.getId}, " +
+ s"model=$mdlId, " +
s"element=${
holders.map(
p ⇒ s"id=${p.elementId}${if (p.synonym.value == null)
"" else s", value=${p.synonym.value}"}"
@@ -357,10 +373,10 @@ object NCDeployManager extends NCService with
DecorateAsScala {
if (foundDups) {
if (!mdl.isDupSynonymsAllowed)
- throw new NCE(s"Duplicated synonyms are not allowed for model
'${mdl.getId}' - check trace messages.")
+ throw new NCE(s"Duplicated synonyms are not allowed for model
'$mdlId' - check trace messages.")
- logger.warn(s"Found duplicate synonyms - check trace logging for
model: ${mdl.getId}")
- logger.warn(s"Duplicates are allowed by '${mdl.getId}' model but
large number may degrade the performance.")
+ logger.warn(s"Found duplicate synonyms - check trace logging for
model: $mdlId")
+ logger.warn(s"Duplicates are allowed by '$mdlId' model but large
number may degrade the performance.")
}
mdl.getMetadata.put(MDL_META_ALL_ALIASES_KEY, allAliases.toSet)
@@ -383,22 +399,13 @@ object NCDeployManager extends NCService with
DecorateAsScala {
addStopWordsStems = addStopWords,
exclStopWordsStems = exclStopWords,
suspWordsStems = suspWords,
- elements = mdl.getElements.asScala.map(elm ⇒ (elm.getId,
elm)).toMap
+ elements = mdl.getElements.asScala.map(elm ⇒ (elm.getId,
elm)).toMap,
+ samples = NCIntentScanner.scanSamples(mdl)
)
}
/**
*
- * @param name
- * @param col
- */
- @throws[NCE]
- private def checkCollection(name: String, col: Any): Unit =
- if (col == null)
- throw new NCE(s"Collection '$name' can be empty but cannot be
null.")
-
- /**
- *
* @param clsName Factory class name.
*/
@throws[NCE]
@@ -877,4 +884,32 @@ object NCDeployManager extends NCService with
DecorateAsScala {
chunks.map(mkChunk)
}
+
+ @throws[NCE]
+ private def checkSynonyms(mdl: NCModel, parser: NCMacroParser): Unit = {
+ val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId →
p.getSynonyms.asScala.flatMap(parser.expand))
+
+ val mdlId = mdl.getId
+
+ mdlSyns.foreach { case (elemId, syns) ⇒
+ val size = syns.size
+
+ if (size == 0)
+ logger.warn(s"Element '$elemId' doesn't have synonyms
[modelId=$mdlId]")
+ else if (size > Config.synonymsWarnValue)
+ logger.warn(s"Element '$elemId' has too many ($size) synonyms.
Make sure this is truly necessary [modelId=$mdlId]")
+
+ val others = mdlSyns.filter {
+ case (otherId, _) ⇒ otherId != elemId
+ }
+
+ val cross = others.filter {
+ case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty
+ }.toMap.keys.mkString(",")
+
+
+ if (cross.nonEmpty)
+ throw new NCE(s"Element has duplicate synonyms with elements
'$cross' [modelId=$mdlId, elementId=$elemId]")
+ }
+ }
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCIntentScanner.scala
similarity index 89%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCIntentScanner.scala
index b55e8c0..9cffa81 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCIntentScanner.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.model.intent.impl
+package org.apache.nlpcraft.probe.mgrs.deploy
import java.lang.reflect.{InvocationTargetException, Method,
ParameterizedType, Type}
import java.util
@@ -23,7 +23,10 @@ import java.util.function.Function
import com.typesafe.scalalogging.LazyLogging
import org.apache.nlpcraft.common._
+import org.apache.nlpcraft.common.makro.NCMacroParser
+import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
import org.apache.nlpcraft.model._
+import org.apache.nlpcraft.model.intent.impl.NCIntentDslCompiler
import org.apache.nlpcraft.model.intent.utils.NCDslIntent
import scala.collection.JavaConverters._
@@ -61,6 +64,8 @@ object NCIntentScanner extends LazyLogging {
CLS_JAVA_OPT
)
+ private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
+
/**
*
* @param cls
@@ -425,12 +430,10 @@ object NCIntentScanner extends LazyLogging {
* @param mdl Model to scan.
*/
@throws[NCE]
- def scanIntentsSamples(mdl: NCModel): NCIntentSamplesScanResult = {
+ def scanSamples(mdl: NCModel): Map[String, Seq[String]] = {
var annFound = false
- val warns = mutable.ArrayBuffer.empty[String]
-
- val res =
+ val samples =
mdl.getClass.getDeclaredMethods.flatMap(method ⇒ {
def mkMethodName: String =
s"${method.getDeclaringClass.getName}#${method.getName}(...)"
@@ -451,8 +454,9 @@ object NCIntentScanner extends LazyLogging {
if (smpAnn != null) {
if (intAnn == null && refAnn == null) {
- warns += s"@NCTestSample annotation without
corresponding @NCIntent or @NCIntentRef annotations " +
- s"in callback: $mkMethodName"
+ logger.warn(
+ "@NCTestSample annotation without
corresponding @NCIntent or @NCIntentRef annotations " +
+ s"[modelId=${mdl.getId},
callback=$mkMethodName]")
None
}
@@ -460,7 +464,10 @@ object NCIntentScanner extends LazyLogging {
val samples = smpAnn.value().toList
if (samples.isEmpty) {
- warns += s"@NCTestSample annotation is empty
in callback: $mkMethodName"
+ logger.warn(
+ "@NCTestSample annotation is empty " +
+ s"[modelId=${mdl.getId},
callback=$mkMethodName]"
+ )
None
}
@@ -469,7 +476,10 @@ object NCIntentScanner extends LazyLogging {
}
}
else {
- warns += s"@NCTestSample annotation is missing in
callback: $mkMethodName"
+ logger.warn(
+ "@NCTestSample annotation is missing " +
+ s"[modelId=${mdl.getId}, callback=$mkMethodName]"
+ )
None
}
@@ -479,8 +489,30 @@ object NCIntentScanner extends LazyLogging {
}).toMap
if (!annFound)
- warns += s"No intents found."
+ logger.warn(s"No intents found [modelId=${mdl.getId}")
+
+ val parser = new NCMacroParser
+
+ mdl.getMacros.asScala.foreach { case (name, str) ⇒
parser.addMacro(name, str) }
+
+ val allSyns: Set[Seq[String]] =
+ mdl.getElements.
+ asScala.
+ flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
+ map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
+ toSet
+
+ samples.
+ flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
+ map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
+ foreach {
+ case (s, sNorm) ⇒
+ val seq: Seq[String] = sNorm.split("
").map(NCNlpPorterStemmer.stem)
+
+ if (!allSyns.exists(_.intersect(seq).nonEmpty))
+ logger.warn(s"Intent sample doesn't contain any direct
synonyms [modelId=${mdl.getId}, sample=$s]")
+ }
- NCIntentSamplesScanResult(res, warns)
+ samples
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCModelWrapper.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCModelWrapper.scala
index d83cdfb..e50a347 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCModelWrapper.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCModelWrapper.scala
@@ -11,7 +11,7 @@ import org.apache.nlpcraft.model.{NCElement, NCModel,
NCVariant}
import org.apache.nlpcraft.probe.mgrs.NCSynonym
import scala.collection.JavaConverters._
-import scala.collection.{Seq, mutable}
+import scala.collection.{Map, Seq, mutable}
/**
*
@@ -32,7 +32,8 @@ case class NCModelWrapper(
addStopWordsStems: Set[String],
exclStopWordsStems: Set[String],
suspWordsStems: Set[String],
- elements: Map[String /*Element ID*/ , NCElement]
+ elements: Map[String /*Element ID*/ , NCElement],
+ samples: Map[String, Seq[String]]
) {
/**
* Makes variants for given sentences.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
deleted file mode 100644
index f73c316..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.{NCE, NCService}
-import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
-import
org.apache.nlpcraft.probe.mgrs.inspections.inspectors.{NCIntentsInspection,
NCMacrosInspection, NCSynonymsInspection}
-
-import scala.concurrent.Future
-
-/**
- * Probe-side inspection manager.
- */
-object NCInspectionManager extends NCService with NCOpenCensusModelStats {
- private final val PROBE_INSPECTIONS =
- Seq(
- NCMacrosInspection,
- NCSynonymsInspection,
- NCIntentsInspection
- )
-
- override def start(parent: Span): NCService = startScopedSpan("start",
parent) { _ ⇒
- PROBE_INSPECTIONS.foreach(_.start(parent))
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit = startScopedSpan("stop", parent) {
_ ⇒
- super.stop()
-
- PROBE_INSPECTIONS.foreach(_.stop(parent))
- }
-
- /**
- *
- * @param mdlId Model ID.
- * @param inspName Inspection ID.
- * @param args Inspection arguments.
- * @param parent Optional parent trace span.
- */
- def inspect(mdlId: String, inspName: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult] =
- startScopedSpan(
- "inspect",
- parent,
- "modelId" → mdlId,
- "inspName" → inspName,
- "args" → args.orNull) { _ ⇒
- PROBE_INSPECTIONS.find(_.getName == inspName) match {
- case Some(insp) ⇒ insp.inspect(mdlId, inspName, args)
- case None ⇒ throw new NCE(s"Unsupported inspection: $inspName")
- }
- }
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
deleted file mode 100644
index 009be47..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import org.apache.nlpcraft.common.inspections.NCInspectionService
-import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
-import org.apache.nlpcraft.model.NCModel
-import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
-
-import scala.collection.JavaConverters._
-import scala.collection._
-
-/**
- * Inspection for model's intents.
- */
-object NCIntentsInspection extends NCProbeInspection {
- private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
-
- override def getName: String = "intents"
-
- override def body(
- mdl: NCModel,
- args: Option[String],
- suggs: mutable.Buffer[String],
- warns: mutable.Buffer[String],
- errs: mutable.Buffer[String]): Unit = {
- if (args.isDefined && args.get.nonEmpty)
- warns += s"Invalid inspection arguments (ignoring): ${args.get}"
-
- val res = NCIntentScanner.scanIntentsSamples(mdl)
-
- val parser = new NCMacroParser
-
- mdl.getMacros.asScala.foreach { case (name, str) ⇒
parser.addMacro(name, str) }
-
- val allSyns: Set[Seq[String]] =
- mdl.getElements.
- asScala.
- flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
- map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
- toSet
-
- res.samples.
- flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
- map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
- foreach {
- case (s, sNorm) ⇒
- val seq: Seq[String] = sNorm.split("
").map(NCNlpPorterStemmer.stem)
-
- if (!allSyns.exists(_.intersect(seq).nonEmpty))
- warns += s"Intent sample '$s' doesn't contain any
direct synonyms."
- }
- }
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
deleted file mode 100644
index 005f520..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import org.apache.nlpcraft.common.inspections.NCInspectionService
-import org.apache.nlpcraft.model.NCModel
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-/**
- * Inspection for model's macros.
- */
-object NCMacrosInspection extends NCProbeInspection {
- override def getName: String = "macros"
-
- override def body(
- mdl: NCModel,
- args: Option[String],
- suggs: mutable.Buffer[String],
- warns: mutable.Buffer[String],
- errs: mutable.Buffer[String]): Unit = {
- if (args.isDefined && args.get.nonEmpty)
- warns += s"Invalid inspection arguments (ignoring): ${args.get}"
-
- val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
-
- warns ++= mdl.getMacros.asScala.keys.flatMap(makro ⇒
- if (syns.exists(_.contains(makro))) None else Some(s"Unused macro:
$makro")
- )
- }
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
deleted file mode 100644
index 69c0ba5..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCProbeInspection.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.common.inspections.{NCInspectionResult,
NCInspectionService}
-import org.apache.nlpcraft.model.NCModel
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-
-import scala.collection.mutable
-import scala.concurrent.Future
-
-trait NCProbeInspection extends NCInspectionService {
- /**
- *
- * @param mdlId
- * @param inspName
- * @param args
- * @param parent
- * @return
- */
- override def inspect(mdlId: String, inspName: String, args:
Option[String], parent: Span = null): Future[NCInspectionResult] = {
- val now = System.currentTimeMillis()
-
- startScopedSpan(
- "inspect",
- parent,
- "modelId" → mdlId,
- "inspName" → inspName,
- "args" → args.orNull) { _ ⇒
- Future {
- val errs = mutable.Buffer.empty[String]
- val warns = mutable.Buffer.empty[String]
- val suggs = mutable.Buffer.empty[String]
-
- NCModelManager.getModelWrapper(mdlId) match {
- case Some(x) ⇒ body(x.proxy, args, suggs, warns, errs)
- case None ⇒ errs += s"Model not found: $mdlId"
- }
-
- NCInspectionResultImpl(
- inspectionId = inspName,
- modelId = mdlId,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- warnings = warns,
- suggestions = suggs,
- errors = errs
- )
- }(getExecutor)
- }
- }
-
- /**
- * Convenient adapter for the probe-side inspection implementation.
- *
- * @param mdl
- * @param args
- * @param suggs Mutable collector for suggestions.
- * @param warns Mutable collector for warnings.
- * @param errs Mutable collector for errors.
- */
- protected def body(
- mdl: NCModel,
- args: Option[String],
- suggs: mutable.Buffer[String],
- warns: mutable.Buffer[String],
- errs: mutable.Buffer[String]
- )
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
deleted file mode 100644
index 77575af..0000000
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import org.apache.nlpcraft.common.inspections.NCInspectionService
-import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.model.NCModel
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-/**
- * Inspection for model's synonyms.
- */
-object NCSynonymsInspection extends NCProbeInspection {
- // Pretty arbitrary number...
- // TODO: make it part of inspection configuration.
- private final val TOO_MANY_SYNS = 10000
-
- override def getName: String = "synonyms"
-
- override def body(
- mdl: NCModel,
- args: Option[String],
- suggs: mutable.Buffer[String],
- warns: mutable.Buffer[String],
- errs: mutable.Buffer[String]): Unit = {
- if (args.isDefined && args.get.nonEmpty)
- warns += s"Invalid inspection arguments (ignoring): ${args.get}"
-
- val parser = new NCMacroParser()
-
- mdl.getMacros.asScala.foreach { case (name, str) ⇒
parser.addMacro(name, str) }
-
- val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId →
p.getSynonyms.asScala.flatMap(parser.expand))
-
- mdlSyns.foreach { case (elemId, syns) ⇒
- val size = syns.size
-
- if (size == 0)
- warns += s"Element '$elemId' doesn't have synonyms."
- else if (size > TOO_MANY_SYNS)
- warns += s"Element '$elemId' has too many ($size) synonyms.
Make sure this is truly necessary."
-
- val others = mdlSyns.filter {
- case (otherId, _) ⇒ otherId != elemId
- }
-
- val cross = others.filter {
- case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty
- }.toMap.keys.mkString(",")
-
- if (cross.nonEmpty)
- errs += s"Element '$elemId' has duplicate synonyms with
elements '$cross'."
- }
- }
-}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index 446c2e4..06130e7 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -17,16 +17,12 @@
package org.apache.nlpcraft.probe.mgrs.model
-import java.util
-
import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.model._
-import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.probe.mgrs.deploy.{NCModelWrapper, _}
-import scala.collection.JavaConverters._
import scala.collection.convert.DecorateAsScala
import scala.util.control.Exception._
@@ -40,7 +36,6 @@ object NCModelManager extends NCService with DecorateAsScala {
// Access mutex.
private final val mux = new Object()
-
@throws[NCE]
override def start(parent: Span = null): NCService =
startScopedSpan("start", parent) { span ⇒
val tbl = NCAsciiTable("Model ID", "Name", "Ver.", "Elements",
"Synonyms")
@@ -104,7 +99,6 @@ object NCModelManager extends NCService with DecorateAsScala
{
super.stop()
}
-
/**
*
* @return
@@ -127,26 +121,4 @@ object NCModelManager extends NCService with
DecorateAsScala {
wrappers.get(mdlId)
}
}
-
- /**
- * TODO:
- * Gets model data which can be transferred between probe and server.
- *
- * @param mdlId Model ID.
- * @param parent
- * @return
- */
- def getModelInfo(mdlId: String, parent: Span = null):
java.util.Map[String, Any] =
- startScopedSpan("getModel", parent, "mdlId" → mdlId) { _ ⇒
- val w = mux.synchronized { wrappers.get(mdlId) }.getOrElse(throw
new NCE(s"Model not found: '$mdlId'"))
- val mdl = w.proxy
-
- val data = new util.HashMap[String, Any]()
-
- data.put("macros", mdl.getMacros)
- data.put("synonyms", mdl.getElements.asScala.map(p ⇒ p.getId →
p.getSynonyms).toMap.asJava)
- data.put("samples",
NCIntentScanner.scanIntentsSamples(mdl).samples.map(p ⇒ p._1 →
p._2.asJava).asJava)
-
- data
- }
}
\ No newline at end of file
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 52d9d1d..444ccc1 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -31,8 +31,6 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.crypto.NCCipher
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.socket.NCSocket
@@ -57,7 +55,6 @@ import scala.util.{Failure, Success}
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private final val TYPE_INSPECTION_RESP = new
TypeToken[NCInspectionResultImpl]() {}.getType
private final val TYPE_MODEL_INFO_RESP = new
TypeToken[util.HashMap[String, AnyRef]]() {}.getType
// Type safe and eager configuration container.
@@ -147,7 +144,6 @@ object NCProbeManager extends NCService {
@volatile private var pool: ExecutorService = _
@volatile private var isStopping: AtomicBoolean = _
- @volatile private var probeInsp: ConcurrentHashMap[String,
Promise[NCInspectionResult]] = _
@volatile private var modelsInfo: ConcurrentHashMap[String,
Promise[java.util.Map[String, AnyRef]]] = _
/**
@@ -169,7 +165,6 @@ object NCProbeManager extends NCService {
isStopping = new AtomicBoolean(false)
- probeInsp = new ConcurrentHashMap[String,
Promise[NCInspectionResult]]()
modelsInfo = new ConcurrentHashMap[String,
Promise[java.util.Map[String, AnyRef]]]()
pool = Executors.newFixedThreadPool(Config.poolSize)
@@ -210,7 +205,6 @@ object NCProbeManager extends NCService {
U.stopThread(dnSrv)
U.stopThread(upSrv)
- probeInsp = null
modelsInfo = null
super.stop()
@@ -698,12 +692,17 @@ object NCProbeManager extends NCService {
promise.success(r)
}
}
-
+
typ match {
case "P2S_PING" ⇒ ()
- case "P2S_PROBE_INSPECTION" ⇒ processPromise(probeInsp,
TYPE_INSPECTION_RESP)
- case "P2S_MODEL_INFO" ⇒ processPromise(modelsInfo,
TYPE_MODEL_INFO_RESP)
+ case "P2S_MODEL_INFO" ⇒
+ val p = modelsInfo.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ p.success(GSON.fromJson(probeMsg.data[String]("resp"),
TYPE_MODEL_INFO_RESP))
+ else
+ logger.warn(s"Message ignored: $probeMsg")
case "P2S_ASK_RESULT" ⇒
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -1007,36 +1006,25 @@ object NCProbeManager extends NCService {
/**
*
* @param mdlId
- * @param inspName
- * @param args
* @param parent
* @return
*/
- def runInspection(mdlId: String, inspName: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("runInspection", parent, "modelId" → mdlId, "inspName"
→ inspName) { _ ⇒
- val params = mutable.HashMap.empty[String, Serializable] ++
Map("mdlId" → mdlId, "inspName" → inspName)
+ def getModelInfo(mdlId: String, parent: Span = null):
Future[java.util.Map[String, AnyRef]] =
+ startScopedSpan("getModelInfo", parent, "modelId" → mdlId) { _ ⇒
+ getProbeForModelId(mdlId) match {
+ case Some(probe) ⇒
+ val msg = NCProbeMessage("S2P_MODEL_INFO", "mdlId" → mdlId)
- if (args.isDefined)
- params += "args" → GSON.toJson(args.get)
+ val p = Promise[java.util.Map[String, AnyRef]]()
- probePromise(parent, mdlId, probeInsp, "S2P_PROBE_INSPECTION",
params.toSeq :_*)
- }
+ modelsInfo.put(msg.getGuid, p)
- /**
- *
- * @param mdlId
- * @param parent
- * @return
- */
- def getModelInfo(mdlId: String, parent: Span = null):
Future[java.util.Map[String, AnyRef]] =
- startScopedSpan("getModelInfo", parent, "modelId" → mdlId) { _ ⇒
- probePromise(
- parent,
- mdlId,
- modelsInfo,
- "S2P_MODEL_INFO",
- "mdlId" → mdlId
- )
+ sendToProbe(probe.probeKey, msg, parent)
+
+ p.future
+
+ case None ⇒ throw new NCE(s"Probe not found for model:
'$mdlId''")
+ }
}
/**
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index bbac7e1..a40d17b 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -164,7 +164,9 @@ object NCSuggestSynonymManager extends NCService {
case Success(m) ⇒
try {
require(
- m.containsKey("macros") &&
m.containsKey("elementsSynonyms") && m.containsKey("intentsSamples")
+ m.containsKey("macros") &&
+ m.containsKey("synonyms") &&
+ m.containsKey("samples")
)
val mdlMacros = m.get("macros").
