This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41 by this push:
new ca40a33 WIP.
ca40a33 is described below
commit ca40a33cd5ad48002971fd359f0f455b110f1666
Author: Sergey Kamov <[email protected]>
AuthorDate: Mon Aug 31 13:25:24 2020 +0300
WIP.
---
.../mgrs/inspections/NCInspectionManager.scala | 2 +-
.../mgrs/inspections/inspectors/NCInspector.scala | 38 +++++++++++++++-
.../inspectors/NCInspectorIntents.scala | 6 +--
.../inspections/inspectors/NCInspectorMacros.scala | 8 ++--
.../inspectors/NCInspectorSynonyms.scala | 6 +--
.../server/inspections/NCInspectionManager.scala | 19 ++++++--
.../inspections/inspectors/NCInspector.scala | 36 ++++++++++++++-
.../inspectors/NCInspectorSuggestions.scala | 51 ++++++++--------------
8 files changed, 115 insertions(+), 51 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
index a2aa3e7..3bc5ad3 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
@@ -26,7 +26,7 @@ import
org.apache.nlpcraft.probe.mgrs.inspections.inspectors.{NCInspectorIntents
import scala.concurrent.Future
/**
- *
+ * TODO:
*/
object NCInspectionManager extends NCService with NCOpenCensusModelStats {
private final val INSPECTORS =
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
index 7ada53e..e175c34 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
@@ -17,11 +17,45 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
+import java.util.concurrent.{ExecutorService, Executors}
+
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.util.NCUtils
+
+import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
-import scala.concurrent.Future
+/**
+ * // TODO: duplicated with server component (same API) etc - should we move
it into common?
+ */
+private[inspections] trait NCInspector extends NCService {
+ @volatile private var pool: ExecutorService = _
+ @volatile protected var executor: ExecutionContextExecutor = _
-private[inspections] trait NCInspector {
+ /**
+ *
+ * @param mdlId
+ * @param inspId
+ * @param args
+ * @param parent
+ * @return
+ */
def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
+
+ override def start(parent: Span): NCService =
+ startScopedSpan("start", parent) { _ ⇒
+ pool = Executors.newCachedThreadPool()
+ executor = ExecutionContext.fromExecutor(pool)
+
+ super.start(parent)
+ }
+
+ override def stop(parent: Span): Unit =
+ startScopedSpan("stop", parent) { _ ⇒
+ super.stop(parent)
+
+ NCUtils.shutdownPools(pool)
+ executor = null
+ }
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
index 31d6352..9983b0d 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
@@ -18,11 +18,11 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCE
import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
-import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
@@ -31,7 +31,7 @@ import scala.collection._
import scala.concurrent.Future
// TODO:
-object NCInspectorIntents extends NCService with NCInspector {
+object NCInspectorIntents extends NCInspector {
private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
@@ -78,6 +78,6 @@ object NCInspectorIntents extends NCService with NCInspector {
warnings = warns,
suggestions = Seq.empty
)
- }(scala.concurrent.ExecutionContext.Implicits.global)
+ }(executor)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
index 1f5e8e6..19ab343 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
@@ -18,9 +18,9 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCE
import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
@@ -28,7 +28,7 @@ import scala.collection.Seq
import scala.concurrent.Future
// TODO:
-object NCInspectorMacros extends NCService with NCInspector {
+object NCInspectorMacros extends NCInspector {
override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
Future {
@@ -40,7 +40,7 @@ object NCInspectorMacros extends NCService with NCInspector {
val warns =
mdl.getMacros.asScala.keys.
- // TODO: is it valid check? (simple contains)
+ // TODO: is it valid check? (simple contains ?)
flatMap(m ⇒ if (syns.exists(_.contains(m))) None else
Some(s"Macro is not used: $m")).
toSeq
@@ -54,6 +54,6 @@ object NCInspectorMacros extends NCService with NCInspector {
warnings = warns,
suggestions = Seq.empty
)
- }(scala.concurrent.ExecutionContext.Implicits.global)
+ }(executor)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
index 78eb001..2011351 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
@@ -18,10 +18,10 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCE
import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
@@ -29,7 +29,7 @@ import scala.collection.{Seq, mutable}
import scala.concurrent.Future
// TODO:
-object NCInspectorSynonyms extends NCService with NCInspector {
+object NCInspectorSynonyms extends NCInspector {
private final val TOO_MANY_SYNS = 10000
override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
@@ -74,6 +74,6 @@ object NCInspectorSynonyms extends NCService with NCInspector
{
warnings = warns,
suggestions = Seq.empty
)
- }(scala.concurrent.ExecutionContext.Implicits.global)
+ }(executor)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
index ed4340f..a95c841 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
@@ -19,7 +19,7 @@ package org.apache.nlpcraft.server.inspections
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionImpl
+import org.apache.nlpcraft.common.inspections.impl.{NCInspectionImpl,
NCInspectionParameterImpl}
import org.apache.nlpcraft.common.inspections.{NCInspection,
NCInspectionResult}
import org.apache.nlpcraft.server.inspections.inspectors.NCInspectorSuggestions
import org.apache.nlpcraft.server.probe.NCProbeManager
@@ -28,10 +28,11 @@ import scala.collection.Map
import scala.concurrent.Future
/**
- *
+ * TODO:
*/
object NCInspectionManager extends NCService {
- private final val INSPECTIONS: Seq[NCInspection] =
+ private final val INSPECTIONS: Seq[NCInspection] = {
+ // TODO:
Seq(
NCInspectionImpl(
id = "macros",
@@ -61,11 +62,21 @@ object NCInspectionManager extends NCService {
id = "suggestions",
name = "suggestions",
synopsis = "suggestions",
- parameters = Seq.empty,
+ parameters = Seq(
+ NCInspectionParameterImpl(
+ id = "minScore",
+ name = "minScore",
+ value = "minScore",
+ valueType = "double",
+ synopsis = "minScore, range between 0 and 1",
+ description = "minScore"
+ )
+ ),
description = "suggestions",
isServerSide = true
)
)
+ }
private final val SRV_INSPECTORS =
Map(
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
index d455ab5..47d8bd4 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
@@ -1,10 +1,42 @@
package org.apache.nlpcraft.server.inspections.inspectors
+import java.util.concurrent.{ExecutorService, Executors}
+
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.util.NCUtils
+
+import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
-import scala.concurrent.Future
+// TODO: duplicated with probe component (same API) etc - should we move it
into common?
+private[inspections] trait NCInspector extends NCService {
+ @volatile private var pool: ExecutorService = _
+ @volatile protected var executor: ExecutionContextExecutor = _
-private[inspections] trait NCInspector {
+ /**
+ *
+ * @param mdlId
+ * @param inspId
+ * @param args
+ * @param parent
+ * @return
+ */
def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
+
+ override def start(parent: Span): NCService =
+ startScopedSpan("start", parent) { _ ⇒
+ pool = Executors.newCachedThreadPool()
+ executor = ExecutionContext.fromExecutor(pool)
+
+ super.start(parent)
+ }
+
+ override def stop(parent: Span): Unit =
+ startScopedSpan("stop", parent) { _ ⇒
+ super.stop(parent)
+
+ NCUtils.shutdownPools(pool)
+ executor = null
+ }
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
index 89973e5..17258ed 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
@@ -19,7 +19,7 @@ package org.apache.nlpcraft.server.inspections.inspectors
import java.util
import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
-import java.util.concurrent.{ConcurrentHashMap, CopyOnWriteArrayList,
CountDownLatch, ExecutorService, Executors, TimeUnit}
+import java.util.concurrent._
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
@@ -30,21 +30,21 @@ import org.apache.http.client.methods.HttpPost
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
+import org.apache.nlpcraft.common.NCE
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
import org.apache.nlpcraft.common.util.NCUtils
-import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.server.probe.NCProbeManager
import scala.collection.JavaConverters._
import scala.collection.{Seq, mutable}
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future,
Promise}
+import scala.concurrent.{ExecutionContextExecutor, Future, Promise}
// TODO: Possible parameter 'minScore' (double 0 .. 1)
-object NCInspectorSuggestions extends NCService with NCInspector {
+object NCInspectorSuggestions extends NCInspector {
// For context word server requests.
private final val MAX_LIMIT: Int = 10000
private final val BATCH_SIZE = 20
@@ -63,9 +63,6 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
val urlOpt: Option[String] =
getStringOpt("nlpcraft.server.ctxword.url")
}
- @volatile private var pool: ExecutorService = _
- @volatile private var executor: ExecutionContextExecutor = _
-
private final val HANDLER: ResponseHandler[Seq[Seq[Suggestion]]] =
(resp: HttpResponse) ⇒ {
val code = resp.getStatusLine.getStatusCode
@@ -88,9 +85,13 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
}
case class Suggestion(word: String, score: Double)
+
case class RequestData(sentence: String, example: String, elementId:
String, index: Int)
+
case class RestRequestSentence(text: String, indexes: util.List[Int])
+
case class RestRequest(sentences: util.List[RestRequestSentence], limit:
Int, min_score: Double)
+
case class Word(word: String, stem: String) {
require(!word.contains(" "), s"Word cannot contains spaces: $word")
require(
@@ -103,7 +104,6 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
)
}
- // TODO:
case class SuggestionResult(
synonym: String,
ctxWorldServerScore: Double,
@@ -111,7 +111,9 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
)
private def split(s: String): Seq[String] = s.split("
").toSeq.map(_.trim).filter(_.nonEmpty)
+
private def toStem(s: String): String =
split(s).map(NCNlpPorterStemmer.stem).mkString(" ")
+
private def toStemWord(s: String): String = NCNlpPorterStemmer.stem(s)
/**
@@ -455,15 +457,15 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
)
)
}
+ }
+ }
+ catch {
+ case e: NCE ⇒ promise.failure(e)
+ case e: Throwable ⇒
+ logger.warn("Unexpected error.", e)
+
+ promise.failure(e)
}
- }
- catch {
- case e: NCE ⇒ promise.failure(e)
- case e: Throwable ⇒
- logger.warn("Unexpected error.", e)
-
- promise.failure(e)
- }
case e: Throwable ⇒
logger.warn(s"Error getting model information: $mdlId", e)
@@ -473,20 +475,5 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
promise.future
}
-
- override def start(parent: Span): NCService =
- startScopedSpan("start", parent) { _ ⇒
- pool = Executors.newCachedThreadPool()
- executor = ExecutionContext.fromExecutor(pool)
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit =
- startScopedSpan("stop", parent) { _ ⇒
- super.stop(parent)
-
- NCUtils.shutdownPools(pool)
- executor = null
- }
}
+
