This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70_NEW
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-70_NEW by this push:
new de914fb WIP.
de914fb is described below
commit de914fbc2efa2f7c2332b1c4e555628449dc2fa1
Author: Sergey Kamov <[email protected]>
AuthorDate: Wed Jun 30 18:20:25 2021 +0300
WIP.
---
.../nlpcraft/model/NCContextWordElementConfig.java | 2 +-
.../apache/nlpcraft/model/NCModelFileAdapter.java | 4 +--
...on.java => NCContextWordElementConfigJson.java} | 2 +-
.../impl/json/NCContextWordModelConfigJson.java | 6 ++---
.../enrichers/ctxword/NCContextWordEnricher.scala | 29 +++++++++++-----------
.../nlpcraft/model/ctxword/NCContextWordSpec.scala | 2 +-
6 files changed, 23 insertions(+), 22 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCContextWordElementConfig.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCContextWordElementConfig.java
index def58e8..6e28fd5 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCContextWordElementConfig.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCContextWordElementConfig.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
// TODO:
public interface NCContextWordElementConfig extends Serializable {
enum NCContextWordElementPolicy {
- MAX, MIN, AVERAGE, ANY
+ ALL, ANY, AVERAGE, MEDIAN
}
NCContextWordElementPolicy getPolicy();
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
index 83f0e87..b25fc6f 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
@@ -269,7 +269,7 @@ abstract public class NCModelFileAdapter extends
NCModelAdapter {
* @param js
* @return
*/
- private static NCContextWordElementConfig
convert(NCContextWordElementScoreJson js) {
+ private static NCContextWordElementConfig
convert(NCContextWordElementConfigJson js) {
return new NCContextWordElementConfig() {
@Override
public NCContextWordElementPolicy getPolicy() {
@@ -317,7 +317,7 @@ abstract public class NCModelFileAdapter extends
NCModelAdapter {
@Override
public Map<String, NCContextWordElementConfig>
getSupportedElements() {
- Map<String, NCContextWordElementScoreJson> m =
js.getSupportedElements();
+ Map<String, NCContextWordElementConfigJson> m =
js.getSupportedElements();
return m != null ?
m.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, p ->
convert(p.getValue()))) :
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementScoreJson.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementConfigJson.java
similarity index 96%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementScoreJson.java
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementConfigJson.java
index df5a8c7..cb701ad 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementScoreJson.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordElementConfigJson.java
@@ -20,7 +20,7 @@ package org.apache.nlpcraft.model.impl.json;
/**
* TODO:
*/
-public class NCContextWordElementScoreJson {
+public class NCContextWordElementConfigJson {
private String policy;
private double score;
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordModelConfigJson.java
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordModelConfigJson.java
index 2588b87..43e846d 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordModelConfigJson.java
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCContextWordModelConfigJson.java
@@ -25,7 +25,7 @@ import java.util.Map;
public class NCContextWordModelConfigJson {
private String[] samples = new String[0];
private boolean useIntentsSamples;
- private Map<String, NCContextWordElementScoreJson> supportedElements;
+ private Map<String, NCContextWordElementConfigJson> supportedElements;
public String[] getSamples() {
return samples;
@@ -39,10 +39,10 @@ public class NCContextWordModelConfigJson {
public void setUseIntentsSamples(boolean useIntentsSamples) {
this.useIntentsSamples = useIntentsSamples;
}
- public Map<String, NCContextWordElementScoreJson> getSupportedElements() {
+ public Map<String, NCContextWordElementConfigJson> getSupportedElements() {
return supportedElements;
}
- public void setSupportedElements(Map<String,
NCContextWordElementScoreJson> supportedElements) {
+ public void setSupportedElements(Map<String,
NCContextWordElementConfigJson> supportedElements) {
this.supportedElements = supportedElements;
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
index 3d2187b..bb8b997 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
@@ -47,7 +47,6 @@ object NCContextWordEnricher extends NCServerEnricher {
private case class ModelProbeKey(probeId: String, modelId: String)
private case class ElementScore(elementId: String, scores: Double*) {
- lazy val maxScore: Double = scores.max // TODO" logic
override def toString: String = s"Element [id=$elementId,
scores=${scores.sortBy(p => -p).mkString(",", "[", "]")}]"
}
private case class ValuesHolder(
@@ -332,8 +331,13 @@ object NCContextWordEnricher extends NCServerEnricher {
val elemScoreVal = elemScore.getScore
policy match {
- case MAX => scores.max >= elemScoreVal
- case MIN => scores.min >= elemScoreVal
+ case MEDIAN =>
+ val sorted = scores.sorted
+ val len2 = sorted.length / 2
+ val median = if (sorted.length % 2 == 0) (sorted(len2) +
sorted(len2 - 1)) / 2 else sorted(len2)
+
+ median >= elemScoreVal
+ case ALL => scores.forall(_ >= elemScoreVal)
case AVERAGE => scores.sum / scores.size >= elemScoreVal
case ANY => scores.exists(_ >= elemScoreVal)
@@ -347,21 +351,18 @@ object NCContextWordEnricher extends NCServerEnricher {
val detected = mutable.HashMap.empty[NCNlpSentenceToken,
mutable.HashSet[ElementScore]]
def add(
- nounTok: NCNlpSentenceToken, elemId: String, senScore:
Double, sampleScore: Double
+ nounTok: NCNlpSentenceToken, elemId: String, scores :
Double*
): Unit = {
- val maxScore = Math.max(senScore, sampleScore)
val tokElems = detected.getOrElseUpdate(nounTok,
mutable.HashSet.empty[ElementScore])
- def mkNew(): ElementScore = ElementScore(elemId, senScore,
sampleScore)
+ def mkElem(seq: Seq[Double]): ElementScore =
ElementScore(elemId, seq.filter(_ > EXCL_MIN_SCORE):_*)
- tokElems.find(_.elementId == elemId) match {
- case Some(saved) =>
- if (maxScore > saved.maxScore) {
- tokElems -= saved
- tokElems += mkNew()
- }
- case None => tokElems += mkNew()
- }
+ tokElems += (
+ tokElems.find(_.elementId == elemId) match {
+ case Some(ex) => mkElem(scores ++ ex.scores)
+ case None => mkElem(scores)
+ }
+ )
}
val nounToks = ns.tokens.filter(t => NOUNS_POS.contains(t.pos))
diff --git
a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/ctxword/NCContextWordSpec.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/ctxword/NCContextWordSpec.scala
index a871b9a..ee50a2a 100644
---
a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/ctxword/NCContextWordSpec.scala
+++
b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/ctxword/NCContextWordSpec.scala
@@ -50,7 +50,7 @@ class NCContextWordSpecModel extends NCModel {
override def getVersion: String = "1.0.0"
val MDL_LEVEL = 0.4
- val MDL_POLICY = MIN
+ val MDL_POLICY = ALL
override def getContextWordModelConfig: Optional[NCContextWordModelConfig]
= {
Optional.of(
