This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41 by this push:
new 33e5eba WIP.
33e5eba is described below
commit 33e5ebae805d80d23dff31e479a11ecd25dcb977
Author: Sergey Kamov <[email protected]>
AuthorDate: Fri Aug 28 14:24:51 2020 +0300
WIP.
---
nlpcraft/src/main/resources/reference.conf | 4 +-
.../nlpcraft/common/inspections/NCInspection.scala | 60 +++--
.../nlpcraft/common/inspections/NCInspector.scala | 2 +-
.../{NCModelImpl.scala => NCModelWrapper.scala} | 2 +-
.../model/intent/impl/NCIntentScanner.scala | 38 ++-
.../model/intent/impl/NCIntentSolver.scala | 4 +-
.../nlpcraft/probe/mgrs/NCModelDecorator.scala | 6 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 8 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 18 +-
.../inspectors/NCInspectorIntents.scala | 11 +-
.../inspections/inspectors/NCInspectorMacros.scala | 8 +-
.../inspectors/NCInspectorSynonyms.scala | 6 +-
.../NCInspectorSynonymsSuggestions.scala | 20 +-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 25 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 4 +-
.../org/apache/nlpcraft/server/NCServer.scala | 2 +-
.../NCServerInspectorManager.scala | 45 ++--
.../NCInspectorSynonymsSuggestions.scala | 274 +++++++++++----------
.../nlpcraft/server/probe/NCProbeManager.scala | 11 +-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 15 +-
20 files changed, 283 insertions(+), 280 deletions(-)
diff --git a/nlpcraft/src/main/resources/reference.conf
b/nlpcraft/src/main/resources/reference.conf
index 1206440..a880f80 100644
--- a/nlpcraft/src/main/resources/reference.conf
+++ b/nlpcraft/src/main/resources/reference.conf
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# Request timeout property. Important for '/ask/sync' request.
-akka.http.server.request-timeout = 60s
+# Request timeout property. Important for '/ask/sync' and '/model/inspect'
requests.
+akka.http.server.request-timeout = 600s
akka.http.server.remote-address-header = on
\ No newline at end of file
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
index da4a65a..20078af 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
@@ -17,45 +17,41 @@
package org.apache.nlpcraft.common.inspections
-import java.util
import scala.collection.JavaConverters._
/**
- * Note that suggestions and data can be simple type or java collections to
be transfer between server and probe as JSON
+ * Note that 'suggestions' and 'data' must be simple type or java collections
to be transfer between server and probe as JSON
*/
case class NCInspection(
- errors: Option[Seq[String]] = None,
- warnings: Option[Seq[String]] = None,
- suggestions: Option[Seq[AnyRef]] = None,
+ errors: java.util.List[String],
+ warnings: java.util.List[String],
+ suggestions: java.util.List[AnyRef],
// Information for next inspection layer.
- data: Option[AnyRef] = None
-) {
- def serialize(): java.util.Map[String, AnyRef] = {
- val m: util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]
-
- m.put("errors", errors.getOrElse(Seq.empty).asJava)
- m.put("warnings", warnings.getOrElse(Seq.empty).asJava)
- m.put("suggestions", suggestions.getOrElse(Seq.empty).asJava)
- m.put("data", data.orNull)
-
- m
- }
-}
-
+ data: AnyRef = None
+)
object NCInspection {
- def deserialize(m: util.Map[String, AnyRef]): NCInspection = {
- def getSeq(name: String): Option[Seq[String]] = {
- val seq = m.get(name).asInstanceOf[java.util.List[String]]
-
- if (seq.isEmpty) None else Some(seq.asScala)
- }
-
- NCInspection(
- errors = getSeq("errors"),
- warnings = getSeq("warnings"),
- suggestions = getSeq("suggestions"),
- data = Option(m.get("data"))
+ def apply(
+ errors: Option[Seq[String]],
+ warnings: Option[Seq[String]],
+ suggestions: Option[Seq[AnyRef]],
+ data: Option[AnyRef]
+ ): NCInspection = {
+ def convert[T](optSeq: Option[Seq[T]]): java.util.List[T] =
optSeq.getOrElse(Seq.empty).asJava
+
+ new NCInspection(
+ errors = convert(errors),
+ warnings = convert(warnings),
+ suggestions = convert(suggestions),
+ data = data.orNull
)
}
-}
+
+ def apply(
+ errors: Option[Seq[String]],
+ warnings: Option[Seq[String]],
+ suggestions: Option[Seq[AnyRef]]
+ ): NCInspection = apply(errors, warnings, suggestions, None)
+
+ def apply(): NCInspection = NCInspection(None, None, None, None)
+}
\ No newline at end of file
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
index 74788b3..824c4f6 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
@@ -20,5 +20,5 @@ package org.apache.nlpcraft.common.inspections
import io.opencensus.trace.Span
trait NCInspector {
- def inspect(mdlId: String, data: Option[AnyRef] = None, parent: Span =
null): NCInspection
+ def inspect(mdlId: String, prevLayerInspection: Option[NCInspection] =
None, parent: Span = null): NCInspection
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala
similarity index 97%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelImpl.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala
index 93fbe7c..f1f0eb5 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/NCModelWrapper.scala
@@ -26,7 +26,7 @@ import org.apache.nlpcraft.model.{NCContext, NCIntentMatch,
NCModel, NCRejection
* @param proxy Mandatory model proxy.
* @param solver Optional solver.
*/
-class NCModelImpl(val proxy: NCModel, val solver: NCIntentSolver) extends
NCModel {
+class NCModelWrapper(val proxy: NCModel, val solver: NCIntentSolver) extends
NCModel {
require(proxy != null)
override def getId: String = proxy.getId
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
index fa42310..617be92 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentScanner.scala
@@ -425,8 +425,32 @@ object NCIntentScanner extends LazyLogging {
*/
@throws[NCE]
def scanIntentsSamples(mdl: NCModel): Map[String, Seq[String]] = {
+ val (res, warns) = validateIntentsSamples0(mdl)
+
+ warns.foreach(w ⇒ logger.warn(w))
+
+ res
+ }
+
+ /**
+ * Scans given model for intent samples.
+ *
+ * @param mdl Model to scan.
+ */
+ @throws[NCE]
+ def validateIntentsSamples(mdl: NCModel): Seq[String] =
validateIntentsSamples0(mdl)._2
+
+ /**
+ * Scans given model for intent samples.
+ *
+ * @param mdl Model to scan.
+ */
+ @throws[NCE]
+ private def validateIntentsSamples0(mdl: NCModel): (Map[String,
Seq[String]], Seq[String]) = {
var annFound = false
+ val warns = mutable.ArrayBuffer.empty[String]
+
val res =
mdl.getClass.getDeclaredMethods.flatMap(method ⇒ {
def mkMethodName: String =
s"${method.getDeclaringClass.getName}#${method.getName}(...)"
@@ -448,8 +472,8 @@ object NCIntentScanner extends LazyLogging {
if (smpAnn != null) {
if (intAnn == null && refAnn == null) {
- logger.warn(s"@NCTestSample annotation without
corresponding @NCIntent or @NCIntentRef annotations " +
- s"in method (ignoring): $mkMethodName")
+ warns += s"@NCTestSample annotation without
corresponding @NCIntent or @NCIntentRef annotations " +
+ s"in method (ignoring): $mkMethodName"
None
}
@@ -457,7 +481,7 @@ object NCIntentScanner extends LazyLogging {
val samples = smpAnn.value().toList
if (samples.isEmpty) {
- logger.warn(s"@NCTestSample annotation is
empty in method (ignoring): $mkMethodName")
+ warns += s"@NCTestSample annotation is empty
in method (ignoring): $mkMethodName"
None
}
@@ -466,7 +490,7 @@ object NCIntentScanner extends LazyLogging {
}
}
else {
- logger.warn(s"@NCTestSample annotation is missing in
method (ignoring): $mkMethodName")
+ warns += s"@NCTestSample annotation is missing in
method (ignoring): $mkMethodName"
None
}
@@ -476,8 +500,8 @@ object NCIntentScanner extends LazyLogging {
}).toMap
if (!annFound)
- logger.warn(s"Model '${mdl.getId}' doesn't have any intents.")
-
- res
+ warns += s"Model '${mdl.getId}' doesn't have any intents."
+
+ (res, warns)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala
index dc6ee04..9d6686a 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolver.scala
@@ -23,7 +23,7 @@ import org.apache.nlpcraft.common.NCException
import org.apache.nlpcraft.common.debug.NCLogHolder
import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace
import org.apache.nlpcraft.common.util.NCUtils
-import org.apache.nlpcraft.model.impl.{NCModelImpl, NCVariantImpl}
+import org.apache.nlpcraft.model.impl.{NCModelWrapper, NCVariantImpl}
import org.apache.nlpcraft.model.{NCContext, NCIntentMatch, NCIntentSkip,
NCRejection, NCResult, NCToken, NCVariant}
import org.apache.nlpcraft.model.intent.utils.NCDslIntent
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
@@ -126,7 +126,7 @@ class NCIntentSolver(intents: List[(NCDslIntent/*Intent*/,
NCIntentMatch ⇒ NCR
res.groups.find(_.termId == termId).flatMap(grp ⇒
Some(grp.tokens)).getOrElse(Nil).asJava
}
- if
(!in.context.getModel.asInstanceOf[NCModelImpl].onMatchedIntent(intentMatch)) {
+ if
(!in.context.getModel.asInstanceOf[NCModelWrapper].onMatchedIntent(intentMatch))
{
logger.info(
s"Model '${ctx.getModel.getId}' triggered rematching
of intents " +
s"by intent '${res.intentId}' on variant
#${res.variantIdx + 1}."
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
index 9e4d238..3763f33 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCModelDecorator.scala
@@ -22,7 +22,7 @@ import java.util
import org.apache.nlpcraft.common.TOK_META_ALIASES_KEY
import org.apache.nlpcraft.common.nlp.NCNlpSentence
-import org.apache.nlpcraft.model.impl.{NCTokenImpl, NCVariantImpl}
+import org.apache.nlpcraft.model.impl.{NCModelWrapper, NCTokenImpl,
NCVariantImpl}
import org.apache.nlpcraft.model.{NCElement, NCModel, NCVariant}
import scala.collection.JavaConverters._
@@ -32,7 +32,6 @@ import scala.language.implicitConversions
/**
*
* @param model Decorated model.
- * @param intentSamples Intents samples.
* @param synonyms Fast-access synonyms map for first phase.
* @param synonymsDsl Fast-access synonyms map for second phase.
* @param additionalStopWordsStems Stemmatized additional stopwords.
@@ -41,8 +40,7 @@ import scala.language.implicitConversions
* @param elements Map of model elements.
*/
case class NCModelDecorator(
- model: NCModel,
- intentSamples: Map[String, Seq[String]],
+ model: NCModelWrapper,
synonyms: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
synonymsDsl: Map[String/*Element ID*/, Map[Int/*Synonym length*/,
Seq[NCSynonym]]], // Fast access map.
additionalStopWordsStems: Set[String],
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 1c358a0..5f08876 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -94,7 +94,7 @@ object NCCommandManager extends NCService {
)
case "S2P_MODEL_INSPECTION" ⇒
- val resJs: util.Map[String, util.Map[String, AnyRef]] =
+ val res =
NCProbeInspectionManager.inspect(
mdlId = msg.data[String]("mdlId"),
types =
@@ -102,13 +102,13 @@ object NCCommandManager extends NCService {
asScala.
map(p ⇒
NCInspectionType.withName(p.toUpperCase)),
span
- ).map { case (typ, inspection) ⇒ typ.toString →
inspection.serialize() }.asJava
+ )
- NCConnectionManager.send(
+ NCConnectionManager.send(
NCProbeMessage(
"P2S_MODEL_INSPECTION",
"reqGuid" → msg.getGuid,
- "resp" → GSON.toJson(resJs)
+ "resp" → GSON.toJson(res.map(p ⇒
p._1.toString → p._2).asJava)
),
span
)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index 59da69a..f1d36d9 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -26,7 +26,7 @@ import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.inspections.NCInspectionType
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.model.factories.basic.NCBasicModelFactory
-import org.apache.nlpcraft.model.impl.NCModelImpl
+import org.apache.nlpcraft.model.impl.NCModelWrapper
import org.apache.nlpcraft.model.intent.impl.{NCIntentScanner, NCIntentSolver}
import org.apache.nlpcraft.probe.mgrs.inspections.NCProbeInspectionManager
import resource.managed
@@ -47,7 +47,7 @@ object NCDeployManager extends NCService with DecorateAsScala
{
private final val ID_REGEX = "^[_a-zA-Z]+[a-zA-Z0-9:-_]*$"
- @volatile private var models: ArrayBuffer[NCModel] = _
+ @volatile private var models: ArrayBuffer[NCModelWrapper] = _
@volatile private var modelFactory: NCModelFactory = _
object Config extends NCConfigurable {
@@ -81,7 +81,7 @@ object NCDeployManager extends NCService with DecorateAsScala
{
* @return
*/
@throws[NCE]
- private def wrap(mdl: NCModel): NCModel = {
+ private def wrap(mdl: NCModel): NCModelWrapper = {
checkCollection("additionalStopWords", mdl.getAdditionalStopWords)
checkCollection("elements", mdl.getElements)
checkCollection("enabledBuiltInTokens", mdl.getEnabledBuiltInTokens)
@@ -109,12 +109,12 @@ object NCDeployManager extends NCService with
DecorateAsScala {
intents.toList.map(x ⇒ (x._1, (z: NCIntentMatch) ⇒
x._2.apply(z)))
)
- new NCModelImpl(mdl, solver)
+ new NCModelWrapper(mdl, solver)
}
else {
logger.warn(s"Model has no intents: $mdlId")
- new NCModelImpl(mdl, null)
+ new NCModelWrapper(mdl, null)
}
}
@@ -149,7 +149,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
* @param clsName Model class name.
*/
@throws[NCE]
- private def makeModel(clsName: String): NCModel =
+ private def makeModel(clsName: String): NCModelWrapper =
try
wrap(
makeModelFromSource(
@@ -184,7 +184,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
* @param jarFile JAR file to extract from.
*/
@throws[NCE]
- private def extractModels(jarFile: File): Seq[NCModel] = {
+ private def extractModels(jarFile: File): Seq[NCModelWrapper] = {
val clsLdr = Thread.currentThread().getContextClassLoader
val classes = mutable.ArrayBuffer.empty[Class[_ <: NCModel]]
@@ -224,7 +224,7 @@ object NCDeployManager extends NCService with
DecorateAsScala {
@throws[NCE]
override def start(parent: Span = null): NCService =
startScopedSpan("start", parent) { _ ⇒
modelFactory = new NCBasicModelFactory
- models = ArrayBuffer.empty[NCModel]
+ models = ArrayBuffer.empty[NCModelWrapper]
// Initialize model factory (if configured).
Config.modelFactoryType match {
@@ -307,5 +307,5 @@ object NCDeployManager extends NCService with
DecorateAsScala {
*
* @return
*/
- def getModels: Seq[NCModel] = models
+ def getModels: Seq[NCModelWrapper] = models
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
index ffab3e5..ee30dce 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
@@ -20,15 +20,16 @@ package
org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
import org.apache.nlpcraft.common.{NCE, NCService}
+import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
object NCInspectorIntents extends NCService with NCInspector {
- override def inspect(mdlId: String, data: Option[AnyRef], parent: Span =
null): NCInspection =
+ override def inspect(mdlId: String, prevLayerInspection:
Option[NCInspection], parent: Span = null): NCInspection =
startScopedSpan("inspect", parent) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: $mdlId")).model
-
- NCInspection(
- // TODO:
+ val warns = NCIntentScanner.validateIntentsSamples(
+ NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model
not found: '$mdlId'")).model.proxy
)
+
+ NCInspection(errors = None, warnings = if (warns.isEmpty) None
else Some(warns), suggestions = None, data = None)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
index 78e54b8..589abda 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
@@ -25,18 +25,18 @@ import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
object NCInspectorMacros extends NCService with NCInspector {
- override def inspect(mdlId: String, data: Option[AnyRef], parent: Span =
null): NCInspection =
+ override def inspect(mdlId: String, prevLayerInspection:
Option[NCInspection], parent: Span = null): NCInspection =
startScopedSpan("inspect", parent) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: $mdlId")).model
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: '$mdlId'")).model
val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
val warns =
mdl.getMacros.asScala.keys.
- // TODO: is it valid check?
+ // TODO: is it valid check? (simple contains)
flatMap(m ⇒ if (syns.exists(_.contains(m))) None else
Some(s"Macro is not used: $m")).
toSeq
- NCInspection(warnings = if (warns.isEmpty) None else Some(warns))
+ NCInspection(errors = None, warnings = if (warns.isEmpty) None
else Some(warns), suggestions = None, data = None)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
index 9a7ed10..f8ba649 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
@@ -29,9 +29,9 @@ import scala.collection.mutable
object NCInspectorSynonyms extends NCService with NCInspector {
private final val TOO_MANY_SYNS = 10000
- override def inspect(mdlId: String, data: Option[AnyRef], parent: Span =
null): NCInspection =
+ override def inspect(mdlId: String, prevLayerInspection:
Option[NCInspection], parent: Span = null): NCInspection =
startScopedSpan("inspect", parent) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: $mdlId")).model
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: '$mdlId'")).model
val warns = mutable.ArrayBuffer.empty[String]
@@ -58,6 +58,6 @@ object NCInspectorSynonyms extends NCService with NCInspector
{
warns += s"Element: '$elemId' has same synonyms with
'$intersects'"
}
- NCInspection(warnings = if (warns.isEmpty) None else Some(warns))
+ NCInspection(errors = None, warnings = if (warns.isEmpty) None
else Some(warns), suggestions = None, data = None)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
index 3a1c228..dadcc28 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
@@ -28,22 +28,16 @@ import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
object NCInspectorSynonymsSuggestions extends NCService with NCInspector {
- override def inspect(mdlId: String, data: Option[AnyRef], parent: Span =
null): NCInspection =
+ override def inspect(mdlId: String, prevLayerInspection:
Option[NCInspection], parent: Span = null): NCInspection =
startScopedSpan("inspect", parent) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: $mdlId")).model
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new
NCE(s"Model not found: '$mdlId'")).model
- val m = new util.HashMap[String, Any]()
+ val data = new util.HashMap[String, Any]()
- m.put("macros", mdl.getMacros)
- m.put("elementsSynonyms", new util.HashMap[String,
util.List[String]](
- mdl.getElements.asScala.map(p ⇒ p.getId →
p.getSynonyms).toMap.asJava
- ))
- m.put("intentsSamples", new util.HashMap[String,
util.List[String]](
- NCIntentScanner.scanIntentsSamples(mdl).toMap.map {
- case (intentId, samples) ⇒ intentId → samples.asJava
- }.asJava
- ))
+ data.put("macros", mdl.getMacros)
+ data.put("elementsSynonyms", mdl.getElements.asScala.map(p ⇒
p.getId → p.getSynonyms).toMap.asJava)
+ data.put("intentsSamples",
NCIntentScanner.scanIntentsSamples(mdl.proxy).map(p ⇒ p._1 →
p._2.asJava).asJava)
- NCInspection(data = Some(m))
+ NCInspection(errors = None, warnings = None, suggestions = None,
data = Some(data))
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index 2538b1e..e195c0a 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -27,6 +27,7 @@ import org.apache.nlpcraft.common.inspections.NCInspectionType
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.model._
+import org.apache.nlpcraft.model.impl.NCModelWrapper
import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.probe.mgrs.NCSynonymChunkKind._
import org.apache.nlpcraft.probe.mgrs.deploy._
@@ -64,7 +65,7 @@ object NCModelManager extends NCService with DecorateAsScala {
/**
* @param mdl Model.
*/
- private def addNewModel(mdl: NCModel): Unit = {
+ private def addNewModel(mdl: NCModelWrapper): Unit = {
require(Thread.holdsLock(mux))
checkModelConfig(mdl)
@@ -112,21 +113,10 @@ object NCModelManager extends NCService with
DecorateAsScala {
val inspections = NCProbeInspectionManager.inspect(mdlId,
NCInspectionType.values.toSeq)
- inspections.foreach { case(_, inspection) ⇒
- inspection.errors match {
- case Some(errs) ⇒ errs.foreach(e ⇒
logger.error(s"Validation error [model=$mdlId, text=$e"))
- case None ⇒ // No-op.
- }
-
- inspection.warnings match {
- case Some(warns) ⇒ warns.foreach(w ⇒
logger.warn(s"Validation warning [model=$mdlId, text=$w"))
- case None ⇒ // No-op.
- }
-
- inspection.suggestions match {
- case Some(sugs) ⇒ sugs.foreach(s ⇒
logger.info(s"Validation suggestion [model=$mdlId, text=$s"))
- case None ⇒ // No-op.
- }
+ inspections.foreach { case(t, i) ⇒
+ i.errors.asScala.foreach(p ⇒ logger.error(s"Validation
error [model=$mdlId, type=$t, text=$p"))
+ i.warnings.asScala.foreach(p ⇒ logger.warn(s"Validation
warning [model=$mdlId, type=$t, text=$p"))
+ i.suggestions.asScala.foreach(p ⇒ logger.info(s"Validation
suggestion [model=$mdlId, type=$t, text=$p"))
}
})
@@ -279,7 +269,7 @@ object NCModelManager extends NCService with
DecorateAsScala {
* @return Model decorator.
*/
@throws[NCE]
- private def verifyAndDecorate(mdl: NCModel, parser: NCMacroParser):
NCModelDecorator = {
+ private def verifyAndDecorate(mdl: NCModelWrapper, parser: NCMacroParser):
NCModelDecorator = {
for (elm ← mdl.getElements)
checkElement(mdl, elm)
@@ -551,7 +541,6 @@ object NCModelManager extends NCService with
DecorateAsScala {
NCModelDecorator(
model = mdl,
- NCIntentScanner.scanIntentsSamples(mdl).toMap,
synonyms = mkFastAccessMap(filter(syns, dsl = false)),
synonymsDsl = mkFastAccessMap(filter(syns, dsl = true)),
additionalStopWordsStems = addStopWords,
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index 7922e4e..5c3b7c1 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -30,7 +30,7 @@ import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.debug.NCLogHolder
import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote}
import org.apache.nlpcraft.model._
-import org.apache.nlpcraft.model.impl.{NCModelImpl, NCTokenLogger}
+import org.apache.nlpcraft.model.impl.{NCModelWrapper, NCTokenLogger}
import org.apache.nlpcraft.model.intent.impl.NCIntentSolverInput
import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
import org.apache.nlpcraft.model.tools.embedded.NCEmbeddedResult
@@ -574,7 +574,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
span.end()
}
- val mdl: NCModelImpl = mdlDec.model.asInstanceOf[NCModelImpl]
+ val mdl: NCModelWrapper = mdlDec.model.asInstanceOf[NCModelWrapper]
val solverIn = new NCIntentSolverInput(ctx)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index 1472c1d..3277039 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -45,7 +45,7 @@ import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.rest.NCRestManager
import org.apache.nlpcraft.server.sql.NCSqlManager
-import org.apache.nlpcraft.server.model.NCServerInspectorManager
+import org.apache.nlpcraft.server.inspections.NCServerInspectorManager
import org.apache.nlpcraft.server.tx.NCTxManager
import org.apache.nlpcraft.server.user.NCUserManager
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCServerInspectorManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala
similarity index 61%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCServerInspectorManager.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala
index 230be8d..8d42c7e 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCServerInspectorManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala
@@ -15,15 +15,16 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.server.model
+package org.apache.nlpcraft.server.inspections
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.inspections.NCInspection
import org.apache.nlpcraft.common.inspections.NCInspectionType._
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionType}
import org.apache.nlpcraft.common.{NCE, NCService}
-import org.apache.nlpcraft.server.model.inspectors._
+import org.apache.nlpcraft.server.inspections.inspectors._
import org.apache.nlpcraft.server.probe.NCProbeManager
+import scala.collection.JavaConverters.asScalaBufferConverter
import scala.collection._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Future, Promise}
@@ -62,28 +63,30 @@ object NCServerInspectorManager extends NCService {
val promise = Promise[Map[NCInspectionType, NCInspection]]()
NCProbeManager.inspect(mdlId, types, parent).onComplete {
- case Success(map) ⇒
- map.map { case (typ, inspectionProbe) ⇒
- val inspectionSrv = INSPECTORS.get(typ) match {
- case Some(inspector) ⇒ inspector.inspect(mdlId,
inspectionProbe.data)
- case None ⇒ NCInspection()
- }
+ case Success(probeRes) ⇒
+ val srvRes =
+ probeRes.map { case (typ, inspectionProbe) ⇒
+ val inspectionSrv = INSPECTORS.get(typ) match {
+ case Some(inspector) ⇒
inspector.inspect(mdlId, Some(inspectionProbe))
+ case None ⇒ NCInspection()
+ }
+
+ def union[T](seq1: java.util.List[T], seq2:
java.util.List[T]): Option[Seq[T]] = {
+ val seq = seq1.asScala ++ seq2.asScala
- def union[T](seq1: Option[Seq[T]], seq2:
Option[Seq[T]]): Option[Seq[T]] = {
- val seq = seq1.getOrElse(Seq.empty) ++
seq2.getOrElse(Seq.empty)
+ if (seq.isEmpty) None else Some(seq)
+ }
- if (seq.isEmpty) None else Some(seq)
+ typ → NCInspection(
+ errors = union(inspectionProbe.errors,
inspectionSrv.errors),
+ warnings = union(inspectionProbe.warnings,
inspectionSrv.warnings),
+ suggestions =
union(inspectionProbe.suggestions, inspectionSrv.suggestions),
+ // Don't need pass this data on last step.
+ data = None
+ )
}
- typ → NCInspection(
- errors = union(inspectionProbe.errors,
inspectionSrv.errors),
- warnings = union(inspectionProbe.warnings,
inspectionSrv.warnings),
- suggestions = union(inspectionProbe.suggestions,
inspectionSrv.suggestions),
- // Don't need pass this data on last step.
- data = None
- )
- }
- promise.success(map)
+ promise.success(srvRes)
case Failure(err) ⇒ throw err
}(global)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/inspectors/NCInspectorSynonymsSuggestions.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
similarity index 56%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/inspectors/NCInspectorSynonymsSuggestions.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
index 46f0f2a..b72a392 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/inspectors/NCInspectorSynonymsSuggestions.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.server.model.inspectors
+package org.apache.nlpcraft.server.inspections.inspectors
import java.util
import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
import java.util.concurrent.{ConcurrentHashMap, CopyOnWriteArrayList,
CountDownLatch, TimeUnit}
-import java.util.{List => JList}
+import java.util.{List ⇒ JList}
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
@@ -133,12 +133,12 @@ object NCInspectorSynonymsSuggestions extends NCService
with NCInspector {
seq
}
- override def inspect(mdlId: String, data: Option[AnyRef], parent: Span =
null): NCInspection =
+ override def inspect(mdlId: String, prevLayerInspection:
Option[NCInspection], parent: Span = null): NCInspection =
startScopedSpan("inspect", parent) { _ ⇒
val m: util.Map[String, AnyRef] =
- data.
- getOrElse(throw new NCE(s"Missed suggestions data for
model: $mdlId")).
- asInstanceOf[util.Map[String, AnyRef]]
+ prevLayerInspection.
+ getOrElse(throw new NCE(s"Missed previous inspection data
for model: $mdlId")).
+ data.asInstanceOf[util.Map[String, AnyRef]]
val macrosJ = m.get("macros").asInstanceOf[util.Map[String,
String]]
val elementsSynonymsJ =
m.get("elementsSynonyms").asInstanceOf[util.Map[String, util.List[String]]]
@@ -153,7 +153,11 @@ object NCInspectorSynonymsSuggestions extends NCService
with NCInspector {
val intentsSamples = intentsSamplesJ.asScala.map(p ⇒ p._1 →
p._2.asScala)
if (intentsSamples.isEmpty)
- NCInspection()
+ NCInspection(
+ errors = Some(Seq(s"Missed intents samples for:
'$mdlId'")),
+ warnings = None,
+ suggestions = None
+ )
else {
val url = s"${Config.urlOpt.getOrElse(throw new NCE("Context
word server is not configured"))}/suggestions"
@@ -187,15 +191,15 @@ object NCInspectorSynonymsSuggestions extends NCService
with NCInspector {
// Note that we don't use system tokenizer, because
ContextWordServer doesn't have this tokenizer.
// We just split examples words with spaces. Also we divide
SEPARATORS as separated words.
val examples =
- intentsSamples.
- flatMap { case (_, samples) ⇒ samples }.
- map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
- map(ex ⇒ {
- val seq = ex.split(" ")
+ intentsSamples.
+ flatMap { case (_, samples) ⇒ samples }.
+ map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
+ map(ex ⇒ {
+ val seq = ex.split(" ")
- seq → seq.map(toStemWord)
- }).
- toMap
+ seq → seq.map(toStemWord)
+ }).
+ toMap
val elemSyns =
elementsSynonyms.map { case (elemId, syns) ⇒ elemId →
syns.flatMap(parser.expand) }.
@@ -246,156 +250,166 @@ object NCInspectorSynonymsSuggestions extends NCService
with NCInspector {
if (noExElems.nonEmpty)
warns +=
"Some elements don't have synonyms in intent samples,
" +
- s"so the service can't suggest any new synonyms
for such elements: [${noExElems.mkString(", ")}]"
+ s"so the service can't suggest any new synonyms for
such elements: [${noExElems.mkString(", ")}]"
val allReqsCnt = allReqs.map(_._2.size).sum
val allSynsCnt = elemSyns.map(_._2.size).sum
- logger.info(s"Data prepared [examples=${examples.size},
synonyms=$allSynsCnt, requests=$allReqsCnt]")
-
- val allSuggs = new ConcurrentHashMap[String,
JList[Suggestion]]()
- val cdl = new CountDownLatch(1)
- val debugs = mutable.HashMap.empty[RequestData,
Seq[Suggestion]]
- val cnt = new AtomicInteger(0)
+ logger.info(s"Data prepared for ContextWord Server
[examples=${examples.size}, synonyms=$allSynsCnt, requests=$allReqsCnt]")
- val client = HttpClients.createDefault
- val err = new AtomicReference[Throwable]()
-
- for ((elemId, reqs) ← allReqs; batch ←
reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
- NCUtils.asFuture(
- _ ⇒ {
- val post = new HttpPost(url)
-
- post.setHeader("Content-Type", "application/json")
+ if (allReqsCnt == 0)
+ NCInspection(
+ errors = Some(Seq(s"Suggestions cannot be prepared:
'$mdlId'. Samples don't contain synonyms")),
+ warnings = None,
+ suggestions = None
+ )
- post.setEntity(
- new StringEntity(
- GSON.toJson(
- RestRequest(
- sentences = batch.map(p ⇒
RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
- // ContextWord server range is (0,
2), input range is (0, 1)
- min_score =
Config.suggestionsMinScore * 2,
- // We set big limit value and in
fact only minimal score is taken into account.
- limit = MAX_LIMIT
- )
- ),
- "UTF-8"
+ else {
+ val allSuggs = new ConcurrentHashMap[String,
JList[Suggestion]]()
+ val cdl = new CountDownLatch(1)
+ val debugs = mutable.HashMap.empty[RequestData,
Seq[Suggestion]]
+ val cnt = new AtomicInteger(0)
+
+ val client = HttpClients.createDefault
+ val err = new AtomicReference[Throwable]()
+
+ for ((elemId, reqs) ← allReqs; batch ←
reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
+ NCUtils.asFuture(
+ _ ⇒ {
+ val post = new HttpPost(url)
+
+ post.setHeader("Content-Type",
"application/json")
+
+ post.setEntity(
+ new StringEntity(
+ GSON.toJson(
+ RestRequest(
+ sentences = batch.map(p ⇒
RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
+ // ContextWord server range is
(0, 2), input range is (0, 1)
+ min_score =
Config.suggestionsMinScore * 2,
+ // We set big limit value and
in fact only minimal score is taken into account.
+ limit = MAX_LIMIT
+ )
+ ),
+ "UTF-8"
+ )
)
- )
- val resps: Seq[Seq[Suggestion]] =
- try
- client.execute(post, HANDLER)
- finally
- post.releaseConnection()
+ val resps: Seq[Seq[Suggestion]] =
+ try
+ client.execute(post, HANDLER)
+ finally
+ post.releaseConnection()
+
+ require(batch.size == resps.size, s"Batch:
${batch.size}, responses: ${resps.size}")
- require(batch.size == resps.size, s"Batch:
${batch.size}, responses: ${resps.size}")
+ batch.zip(resps).foreach { case (req, resp) ⇒
debugs += req → resp }
- batch.zip(resps).foreach { case (req, resp) ⇒
debugs += req → resp }
+ val i = cnt.addAndGet(batch.size)
- val i = cnt.addAndGet(batch.size)
+ logger.debug(s"Executed: $i requests...")
- logger.info(s"Executed: $i requests...")
+ allSuggs.
+ computeIfAbsent(elemId, (_: String) ⇒ new
CopyOnWriteArrayList[Suggestion]()).
+ addAll(resps.flatten.asJava)
- allSuggs.
- computeIfAbsent(elemId, (_: String) ⇒ new
CopyOnWriteArrayList[Suggestion]()).
- addAll(resps.flatten.asJava)
+ if (i == allReqsCnt)
+ cdl.countDown()
+ },
+ (e: Throwable) ⇒ {
+ err.compareAndSet(null, e)
- if (i == allReqsCnt)
cdl.countDown()
- },
- (e: Throwable) ⇒ {
- err.compareAndSet(null, e)
+ },
+ (_: Unit) ⇒ ()
+ )
+ }
- cdl.countDown()
- },
- (_: Unit) ⇒ ()
- )
- }
+ cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
- cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
+ if (err.get() != null)
+ throw new NCE("Error during work with ContextWord
Server", err.get())
- if (err.get() != null)
- throw new NCE("Error during work with ContextWord Server",
err.get())
+ val allSynsStems =
elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
- val allSynsStems =
elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
+ val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1 →
p._2.asScala).filter(_._2.nonEmpty)
- val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1 →
p._2.asScala).filter(_._2.nonEmpty)
+ val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[SuggestionResult]]
- val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[SuggestionResult]]
+ nonEmptySuggs.
+ foreach { case (elemId, elemSuggs) ⇒
+ elemSuggs.
+ map(sugg ⇒ (sugg, toStem(sugg.word))).
+ groupBy { case (_, stem) ⇒ stem }.
+ // Drops already defined.
+ filter { case (stem, _) ⇒
!allSynsStems.contains(stem) }.
+ map { case (_, group) ⇒
+ val seq = group.map { case (sugg, _) ⇒
sugg }.sortBy(-_.score)
- nonEmptySuggs.
- foreach { case (elemId, elemSuggs) ⇒
- elemSuggs.
- map(sugg ⇒ (sugg, toStem(sugg.word))).
- groupBy { case (_, stem) ⇒ stem }.
- // Drops already defined.
- filter { case (stem, _) ⇒
!allSynsStems.contains(stem) }.
- map { case (_, group) ⇒
- val seq = group.map { case (sugg, _) ⇒ sugg
}.sortBy(-_.score)
+ // Drops repeated.
+ (seq.head, seq.length)
+ }.
+ toSeq.
+ map { case (sugg, cnt) ⇒ (sugg, cnt,
sugg.score * cnt / elemSuggs.size) }.
+ sortBy { case (_, _, sumFactor) ⇒ -sumFactor }.
+ zipWithIndex.
+ foreach { case ((sugg, cnt, _), _) ⇒
+ val seq =
+ res.get(elemId) match {
+ case Some(seq) ⇒ seq
+ case None ⇒
+ val buf =
mutable.ArrayBuffer.empty[SuggestionResult]
- // Drops repeated.
- (seq.head, seq.length)
- }.
- toSeq.
- map { case (sugg, cnt) ⇒ (sugg, cnt, sugg.score *
cnt / elemSuggs.size) }.
- sortBy { case (_, _, sumFactor) ⇒ -sumFactor }.
- zipWithIndex.
- foreach { case ((sugg, cnt, _), _) ⇒
- val seq =
- res.get(elemId) match {
- case Some(seq) ⇒ seq
- case None ⇒
- val buf =
mutable.ArrayBuffer.empty[SuggestionResult]
+ res += elemId → buf
- res += elemId → buf
+ buf
+ }
- buf
- }
+ seq += SuggestionResult(sugg.word,
sugg.score, cnt)
+ }
+ }
- seq += SuggestionResult(sugg.word, sugg.score,
cnt)
- }
- }
+ logger.whenDebugEnabled({
+ logger.debug("Request information:")
- logger.whenInfoEnabled({
- var i = 1
+ var i = 1
- debugs.groupBy(_._1.example).foreach { case (_, m) ⇒
- m.toSeq.sortBy(_._1.sentence).foreach { case (req,
suggs) ⇒
- val s =
- split(req.sentence).
- zipWithIndex.map { case (w, i) ⇒ if (i ==
req.index) s"<<<$w>>>" else w }.
- mkString(" ")
+ debugs.groupBy(_._1.example).foreach { case (_, m) ⇒
+ m.toSeq.sortBy(_._1.sentence).foreach { case (req,
suggs) ⇒
+ val s =
+ split(req.sentence).
+ zipWithIndex.map { case (w, i) ⇒ if (i
== req.index) s"<<<$w>>>" else w }.
+ mkString(" ")
- logger.info(
- s"$i. " +
- s"Request=$s, " +
-
s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
- s"element=${req.elementId}"
- )
+ logger.debug(
+ s"$i. " +
+ s"Request=$s, " +
+
s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
+ s"element=${req.elementId}"
+ )
- i = i + 1
+ i = i + 1
+ }
}
- }
- })
+ })
- val resJs: util.Map[String, JList[util.HashMap[String, Any]]] =
- res.map { case (id, data) ⇒
- id → data.map(d ⇒ {
- val m = new util.HashMap[String, Any]()
+ val resJs: util.Map[String, JList[util.HashMap[String,
Any]]] =
+ res.map { case (id, data) ⇒
+ id → data.map(d ⇒ {
+ val m = new util.HashMap[String, Any]()
- m.put("synonym", d.synonym)
- m.put("ctxWorldServerScore", d.ctxWorldServerScore)
- m.put("suggestedCount", d.suggestedCount)
+ m.put("synonym", d.synonym)
+ m.put("ctxWorldServerScore",
d.ctxWorldServerScore)
+ m.put("suggestedCount", d.suggestedCount)
- m
- }).asJava
- }.asJava
+ m
+ }).asJava
+ }.asJava
- NCInspection(
- warnings = if (warns.isEmpty) None else Some(warns),
- suggestions = Some(Seq(resJs))
- )
+ NCInspection(
+ errors = None, warnings = if (warns.isEmpty) None else
Some(warns), suggestions = Some(Seq(resJs))
+ )
+ }
}
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 525ff5a..b9c4d8c 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -73,7 +73,7 @@ import scala.util.{Failure, Success}
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private val TYPE_INSPECTION_RESP = new TypeToken[ util.Map[String,
util.Map[String, AnyRef]]]() {}.getType
+ private val TYPE_INSPECTION_RESP = new TypeToken[util.Map[String,
NCInspection]]() {}.getType
// Type safe and eager configuration container.
private[probe] object Config extends NCConfigurable {
@@ -706,15 +706,10 @@ object NCProbeManager extends NCService {
val promise =
inspections.remove(probeMsg.data[String]("reqGuid"))
if (promise != null) {
- val respJs: util.Map[String, util.Map[String, AnyRef]]
=
+ val respJs: util.Map[String, NCInspection] =
GSON.fromJson(probeMsg.data[String]("resp"),
TYPE_INSPECTION_RESP)
- val resp =
- respJs.asScala.map { case (k, v) ⇒
- NCInspectionType.withName(k.toUpperCase) →
NCInspection.deserialize(v)
- }
-
- promise.success(resp)
+ promise.success(respJs.asScala.map(p ⇒
NCInspectionType.withName(p._1) → p._2))
}
case "P2S_ASK_RESULT" ⇒
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 9f5f992..96bdaa4 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -17,8 +17,6 @@
package org.apache.nlpcraft.server.rest
-import java.util
-
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.model.HttpMethods._
import akka.http.scaladsl.model._
@@ -36,7 +34,7 @@ import
org.apache.nlpcraft.server.apicodes.NCApiStatusCode.{API_OK, _}
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
import org.apache.nlpcraft.server.mdo.{NCQueryStateMdo, NCUserMdo}
-import org.apache.nlpcraft.server.model.NCServerInspectorManager
+import org.apache.nlpcraft.server.inspections.NCServerInspectorManager
import org.apache.nlpcraft.server.opencensus.NCOpenCensusServerStats
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.query.NCQueryManager
@@ -631,12 +629,6 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
protected def inspect$(reqJs: JsValue): Future[String] = {
val obj = reqJs.asJsObject()
- def getOpt[T](name: String, convert: JsValue ⇒ T): Option[T] =
- obj.fields.get(name) match {
- case Some(v) ⇒ Some(convert(v))
- case None ⇒ None
- }
-
val acsTok = obj.fields("acsTok").convertTo[String]
val mdlId = obj.fields("mdlId").convertTo[String]
val types = obj.fields("types").convertTo[Seq[String]]
@@ -666,13 +658,10 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
inspect(mdlId, typesVals, span).collect {
// We have to use GSON (not spray) here to serialize
`result` field.
case res ⇒
- val m = new util.HashMap[String, AnyRef](
- res.map { case (typ, inspection) ⇒ typ.toString →
inspection.serialize() }.asJava
- )
GSON.toJson(
Map(
"status" → API_OK.toString,
- "result" → m
+ "result" → res.asJava
).asJava
)
}
