This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-387 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 2004cee066aba37e88cfa5725c0523cda5f607f2 Author: Sergey Kamov <[email protected]> AuthorDate: Mon Aug 2 17:04:53 2021 +0300 Fuzzy numerics logic implemented. Minor tests fixes. --- nlpcraft-examples/alarm/pom.xml | 8 ++ .../apache/nlpcraft/examples/alarm/AlarmModel.java | 88 +++++++++++++--------- .../common/nlp/numeric/NCNumericManager.scala | 51 +++++++++---- .../nlp/enrichers/numeric/NCNumericEnricher.scala | 1 - .../scala/org/apache/nlpcraft/NCTestContext.scala | 15 ++++ 5 files changed, 112 insertions(+), 51 deletions(-) diff --git a/nlpcraft-examples/alarm/pom.xml b/nlpcraft-examples/alarm/pom.xml index 84cb0dc..0f6af79 100644 --- a/nlpcraft-examples/alarm/pom.xml +++ b/nlpcraft-examples/alarm/pom.xml @@ -49,6 +49,14 @@ <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>nlpcraft</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java b/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java index 6513e0a..bff4f10 100644 --- a/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java +++ b/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java @@ -17,12 +17,23 @@ package org.apache.nlpcraft.examples.alarm; -import org.apache.nlpcraft.model.*; -import java.time.*; -import java.time.format.*; -import java.util.*; +import org.apache.nlpcraft.model.NCIntentMatch; +import org.apache.nlpcraft.model.NCIntentRef; +import org.apache.nlpcraft.model.NCIntentSampleRef; +import org.apache.nlpcraft.model.NCIntentTerm; +import org.apache.nlpcraft.model.NCModelFileAdapter; +import org.apache.nlpcraft.model.NCRejection; +import org.apache.nlpcraft.model.NCResult; +import org.apache.nlpcraft.model.NCToken; -import static java.time.temporal.ChronoUnit.*; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; + +import static java.time.temporal.ChronoUnit.MILLIS; /** * Alarm example data model. @@ -57,23 +68,50 @@ public class AlarmModel extends NCModelFileAdapter { NCIntentMatch ctx, @NCIntentTerm("nums") List<NCToken> numToks ) { - long unitsCnt = numToks.stream().map(tok -> (String)tok.meta("num:unit")).distinct().count(); + long ms = calculateTime(numToks); - if (unitsCnt != numToks.size()) - throw new NCRejection("Ambiguous time units."); + assert ms >= 0; + timer.schedule( + new TimerTask() { + @Override + public void run() { + System.out.println( + "BEEP BEEP BEEP for: " + ctx.getContext().getRequest().getNormalizedText() + "" + ); + } + }, + ms + ); + + return NCResult.text("Timer set for: " + FMT.format(LocalDateTime.now().plus(ms, MILLIS))); + } + + @Override + public void onDiscard() { + // Clean up when model gets discarded (e.g. during testing). + timer.cancel(); + } + + /** + * TODO: It is implemented as specail method just for tests reasons. + * + * @param numToks + * @return + */ + public static long calculateTime(List<NCToken> numToks) { LocalDateTime now = LocalDateTime.now(); LocalDateTime dt = now; - + for (NCToken num : numToks) { String unit = num.meta("nlpcraft:num:unit"); - + // Skip possible fractionals to simplify. long v = ((Double)num.meta("nlpcraft:num:from")).longValue(); - + if (v <= 0) throw new NCRejection("Value must be positive: " + unit); - + switch (unit) { case "second": { dt = dt.plusSeconds(v); break; } case "minute": { dt = dt.plusMinutes(v); break; } @@ -82,35 +120,13 @@ public class AlarmModel extends NCModelFileAdapter { case "week": { dt = dt.plusWeeks(v); break; } case "month": { dt = dt.plusMonths(v); break; } case "year": { dt = dt.plusYears(v); break; } - + default: // It shouldn't be an assert, because 'datetime' unit can be extended. throw new NCRejection("Unsupported time unit: " + unit); } } - - long ms = now.until(dt, MILLIS); - - assert ms >= 0; - - timer.schedule( - new TimerTask() { - @Override - public void run() { - System.out.println( - "BEEP BEEP BEEP for: " + ctx.getContext().getRequest().getNormalizedText() + "" - ); - } - }, - ms - ); - - return NCResult.text("Timer set for: " + FMT.format(dt)); - } - @Override - public void onDiscard() { - // Clean up when model gets discarded (e.g. during testing). - timer.cancel(); + return now.until(dt, MILLIS); } } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala index e20a18b..1b97003 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala @@ -17,26 +17,15 @@ package org.apache.nlpcraft.common.nlp.numeric -import java.text.{DecimalFormat, ParseException} -import java.util.Locale - import io.opencensus.trace.Span import org.apache.nlpcraft.common.NCService import org.apache.nlpcraft.common.nlp._ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.util.NCUtils.mapResource -case class NCNumericUnit(name: String, unitType: String) -case class NCNumeric( - tokens: Seq[NCNlpSentenceToken], - value: Double, - isFractional: Boolean, - unit: Option[NCNumericUnit] -) +import java.text.{DecimalFormat, ParseException} +import java.util.Locale -/** - * Numeric detection manager. - */ object NCNumericManager extends NCService { // Sets EN numeric format. Locale.setDefault(Locale.forLanguageTag("EN")) @@ -119,6 +108,34 @@ object NCNumericManager extends NCService { } /** + * Tries to find special cases for numerics definition, without explicit numerics. + * + * Example: 'in an hour'. + * + * @param ns Sentence. + */ + private def findFuzzy(ns: NCNlpSentence): Seq[NCNumeric] = { + val senToks: Seq[NCNlpSentenceToken] = ns.tokens.toSeq + val senWords: Seq[String] = senToks.map(_.normText) + + NCFuzzyNumericsConfig.NUMS.map { case (txt, period) => txt.split(" ") -> period }.flatMap { + case (dtWords, dtPeriod) => + senWords.indexOfSlice(dtWords) match { + case -1 => None + case idx => + Some( + NCNumeric( + tokens = senToks.slice(idx, idx + dtWords.length), + value = dtPeriod.value, + isFractional = false, + unit = Some(dtPeriod.unit) + ) + ) + } + } + } + + /** * * @param parent Optional parent span. */ @@ -434,7 +451,13 @@ object NCNumericManager extends NCService { val usedToks = nums.flatMap(_.tokens) - (nums ++ ns.filter(t => !usedToks.contains(t)).flatMap(mkSolidNumUnit)).sortBy(_.tokens.head.index).distinct + val res = + (nums ++ ns.filter(t => !usedToks.contains(t)).flatMap(mkSolidNumUnit)).sortBy(_.tokens.head.index).distinct + + val resToks = res.flatMap(_.tokens) + + // Adds detected special datetime cases with lowest priority. + res ++ findFuzzy(ns).filter(p => !resToks.exists(p.tokens.contains)) } } } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala index e2cfb5c..b28f198 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala @@ -438,7 +438,6 @@ object NCNumericEnricher extends NCServerEnricher { num.tokens.foreach(_.add(note)) } - } } } \ No newline at end of file diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala index 0201315..d0b3ceb 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala @@ -165,6 +165,21 @@ abstract class NCTestContext { assertEquals(expResp, resExtractor(res.getResult.get)) } + /** + * + * @param req + * @param resExtractor + * @param validator + * @tparam T + */ + protected def checkResult[T](req: String, resExtractor: String => T, validator: T => Boolean): Unit = { + val res = getClient.ask(req) + + assertTrue(res.isOk, s"Unexpected result, error=${res.getResultError.orElse(null)}") + assertTrue(res.getResult.isPresent) + assertTrue(validator.apply(resExtractor(res.getResult.get))) + } + final protected def getClient: NCTestClient = { if (cli == null) throw new IllegalStateException("Client is not started.")
