This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-404
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-404 by this push:
new 6ed87c7 WIP on NLPCRAFT-404
6ed87c7 is described below
commit 6ed87c7b1ef00f1a5b0e071101ba3436671ba7db
Author: Aaron Radzinski <[email protected]>
AuthorDate: Sat Aug 7 21:50:14 2021 -0700
WIP on NLPCRAFT-404
---
.../model/intent/solver/NCIntentSolver.scala | 2 +-
.../model/intent/solver/NCIntentSolverEngine.scala | 4 +-
.../model/tools/test/NCTestAutoModelValidator.java | 70 ++++++++++++++++++----
.../test/impl/NCTestAutoModelValidatorImpl.scala | 25 +++++---
4 files changed, 78 insertions(+), 23 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
index 2e59fc1..88af005 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
@@ -145,7 +145,7 @@ class NCIntentSolver(intents: List[(NCIdlIntent/*Intent*/,
NCIntentMatch => NCRe
if (cbRes.getIntentId == null)
cbRes.setIntentId(res.intentId)
- logger.info(s"Intent '${res.intentId}' for variant
#${res.variantIdx + 1} selected as the ${g(bo("'best match'"))}.")
+ logger.info(s"Intent '${res.intentId}' for variant
#${res.variantIdx + 1} selected as the ${g(bo("<|best match|>"))}.")
NCDialogFlowManager.addMatchedIntent(
intentMatch,
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolverEngine.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolverEngine.scala
index d8a3207..37475ac 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolverEngine.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolverEngine.scala
@@ -323,11 +323,11 @@ object NCIntentSolverEngine extends LazyLogging with
NCOpenCensusTrace {
tbl += (
Seq(
s"#${m.variantIdx + 1}",
- g(bo("best match"))
+ g(bo("<|best match|>"))
),
Seq(
im.intent.id,
- g(bo("best match"))
+ g(bo("<|best match|>"))
),
mkPickTokens(im),
w.toAnsiString
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.java
index 4648cac..3443f7d 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.java
@@ -20,6 +20,10 @@ package org.apache.nlpcraft.model.tools.test;
import org.apache.nlpcraft.model.*;
import org.apache.nlpcraft.model.tools.test.impl.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
/**
* Data model auto-validator is based on {@link NCIntentSample} and {@link
NCIntentSampleRef} annotations. Validation
* consists of starting an embedded probe, scanning all deployed models for
these annotations and their
@@ -55,13 +59,28 @@ import org.apache.nlpcraft.model.tools.test.impl.*;
* @see NCIntentRef
*/
public class NCTestAutoModelValidator {
- /** Optional, comma-separate list of class names for the models to test. */
+ /**
+ * System property name for the optional comma-separate list of class
names for the models to test.
+ * If not provided, the models from the probe configuration will be used.
+ */
public final static String PROP_MODELS = "NLPCRAFT_TEST_MODELS";
/**
+ * System property name for the optional path to probe configuration file.
If not provided,
+ * the default probe configuration file will be used.
+ */
+ public final static String PROP_PROBE_CFG = "NLPCRAFT_PROBE_CONFIG";
+
+ /**
+ * System property name for the optional comma separated list of intent
IDs to test.
+ * If not provided, all detected intents will be tested.
+ */
+ public final static String PROP_INTENT_IDS = "NLPCRAFT_TEST_INTENT_IDS";
+
+ /**
* Performs validation based on {@link NCIntentSample} and {@link
NCIntentSampleRef} annotations.
* <p>
- * This is an entry point for a standalone application that expects two
system properties (both optional):
+ * This is an entry point for a standalone application that expects three
system properties (all optional):
* <ul>
* <li>
* <code>NLPCRAFT_TEST_MODELS</code> - optional comma separated
list of fully qualified data model class names to
@@ -73,21 +92,23 @@ public class NCTestAutoModelValidator {
* <code>NLPCRAFT_PROBE_CONFIG</code> - optional path to probe
configuration file. If not provided -
* the default NLPCraft configuration will be used.
* </li>
+ * <li>
+ * <code>NLPCRAFT_TEST_INTENT_IDS</code> - optional comma
separated list of intent IDs to test. If
+ * not provided, all detected intents will be tested.
+ * </li>
* </ul>
* Note that standard validation output will be printed out to the
configured logger (e.g. log4j), if any.
*
* @param args These arguments are ignored.
- * @throws Exception Thrown in case of any unexpected errors during
validation. Note that standard validation
- * output will be printed out to the configured logger.
*/
- public static void main(String[] args) throws Exception {
+ public static void main(String[] args) {
System.exit(NCTestAutoModelValidatorImpl.isValid() ? 0 : 1);
}
/**
* Performs validation based on {@link NCIntentSample} and {@link
NCIntentSampleRef} annotations.
* <p>
- * This method accepts two system properties (both optional):
+ * This method expects three system properties (all optional):
* <ul>
* <li>
* <code>NLPCRAFT_TEST_MODELS</code> - optional comma separated
list of fully qualified data model class names to
@@ -99,15 +120,17 @@ public class NCTestAutoModelValidator {
* <code>NLPCRAFT_PROBE_CONFIG</code> - optional path to probe
configuration file. If not provided -
* the default NLPCraft configuration will be used.
* </li>
+ * <li>
+ * <code>NLPCRAFT_TEST_INTENT_IDS</code> - optional comma
separated list of intent IDs to test. If
+ * not provided, all detected intents will be tested.
+ * </li>
* </ul>
* Note that standard validation output will be printed out to the
configured logger (e.g. log4j), if any.
*
* @return <code>True</code> if no validation errors found,
<code>false</code> otherwise. Note that
* standard validation output will be printed out to the configured
logger (e.g. log4j), if any.
- * @throws Exception Thrown in case of any unexpected errors during
validation. Note that standard validation
- * output will be printed out to the configured logger (e.g. log4j),
if any.
*/
- public static boolean isValid() throws Exception {
+ public static boolean isValid() {
return NCTestAutoModelValidatorImpl.isValid();
}
@@ -120,12 +143,35 @@ public class NCTestAutoModelValidator {
* @param claxx Data model class.
* @return <code>True</code> if no validation errors found,
<code>false</code> otherwise. Note that
* standard validation output will be printed out to the configured
logger (e.g. log4j), if any.
- * @throws Exception Thrown in case of any unexpected errors during
validation. Note that standard validation
- * output will be printed out to the configured logger (e.g. log4j),
if any.
*/
- public static boolean isValid(Class<NCModel> claxx) throws Exception {
+ public static boolean isValid(Class<NCModel> claxx) {
+ assert claxx != null;
+
System.setProperty(PROP_MODELS, claxx.getName());
return NCTestAutoModelValidatorImpl.isValid();
}
+
+ /**
+ * Performs validation based on {@link NCIntentSample} and {@link
NCIntentSampleRef} annotations for given model.
+ * <p>
+ * This is a convenient shortcut that is equivalent to setting
<code>NLPCRAFT_TEST_MODELS</code> system
+ * property (overriding any existing value) with given mode class name and
calling {@link #isValid()} method.
+ *
+ * @param claxx Data model class.
+ * @param intentIds Collection of intent IDs to test. If this list is
empty or <code>null</code>, all
+ * intents will be tested.
+ * @return <code>True</code> if no validation errors found,
<code>false</code> otherwise. Note that
+ * standard validation output will be printed out to the configured
logger (e.g. log4j), if any.
+ */
+ public static boolean isValid(Class<NCModel> claxx, Collection<String>
intentIds) {
+ assert claxx != null;
+
+ System.setProperty(PROP_MODELS, claxx.getName());
+
+ if (intentIds != null && !intentIds.isEmpty())
+ System.setProperty(PROP_INTENT_IDS, String.join(",", intentIds));
+
+ return NCTestAutoModelValidatorImpl.isValid();
+ }
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
index 780bbb4..8cec3ee 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala
@@ -21,7 +21,8 @@ import com.typesafe.scalalogging.LazyLogging
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.model.tools.embedded.NCEmbeddedProbe
-import org.apache.nlpcraft.model.tools.test.{NCTestAutoModelValidator,
NCTestClientBuilder}
+import org.apache.nlpcraft.model.tools.test.NCTestClientBuilder
+import org.apache.nlpcraft.model.tools.test.NCTestAutoModelValidator._
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.jdk.CollectionConverters.SeqHasAsJava
@@ -30,16 +31,13 @@ import scala.jdk.CollectionConverters.SeqHasAsJava
* Implementation for `NCTestAutoModelValidator` class.
*/
private [test] object NCTestAutoModelValidatorImpl extends LazyLogging {
- private final val PROP_PROBE_CFG = "NLPCRAFT_PROBE_CONFIG"
-
/**
*
* @throws Exception Thrown in case of any errors.
* @return
*/
- @throws[Exception]
def isValid: Boolean = {
- val prop = NCTestAutoModelValidator.PROP_MODELS
+ val prop = PROP_MODELS
val classes = U.sysEnv(prop) match {
case Some(s) => U.splitTrimFilter(s, ",")
@@ -49,10 +47,14 @@ private [test] object NCTestAutoModelValidatorImpl extends
LazyLogging {
null
}
val cfgFile = U.sysEnv(PROP_PROBE_CFG).orNull
+ val intentIds = U.sysEnv(PROP_INTENT_IDS).flatMap(s =>
Some(s.split(",").map(_.strip).toSeq))
if (NCEmbeddedProbe.start(cfgFile, classes.asJava))
try
- process(NCModelManager.getAllModels().map(p => p.model.getId
-> p.samples.toMap).toMap.filter(_._2.nonEmpty))
+ process(
+ NCModelManager.getAllModels().map(p => p.model.getId ->
p.samples.toMap).toMap.filter(_._2.nonEmpty),
+ intentIds
+ )
finally
NCEmbeddedProbe.stop()
else
@@ -62,9 +64,13 @@ private [test] object NCTestAutoModelValidatorImpl extends
LazyLogging {
/**
*
* @param samples
+ * @param intentIds
* @return
*/
- private def process(samples: Map[/*Model ID*/String, Map[String/*Intent
ID*/, Seq[Seq[String]]/*Samples*/]]): Boolean = {
+ private def process(
+ samples: Map[/*Model ID*/String, Map[String/*Intent ID*/,
Seq[Seq[String]]/*Samples*/]],
+ intentIds: Option[Seq[String]]
+ ): Boolean = {
case class Result(
modelId: String,
intentId: String,
@@ -100,7 +106,10 @@ private [test] object NCTestAutoModelValidatorImpl extends
LazyLogging {
cli.close()
}
- for ((intentId, seq) <- samples; txts <- seq) yield ask(intentId,
txts)
+ intentIds match {
+ case Some(ids) => for ((intentId, seq) <- samples if
ids.contains(intentId); txts <- seq) yield ask(intentId, txts)
+ case None => for ((intentId, seq) <- samples; txts <- seq)
yield ask(intentId, txts)
+ }
}.flatMap(_.toSeq)
val tbl = NCAsciiTable()
