WIP.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index f63a088..26d29c1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -17,11 +17,12 @@
package org.apache.nlpcraft.probe.mgrs.cmd
-import com.google.gson.Gson
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.{NCService, _}
-import org.apache.nlpcraft.model.NCToken
+import org.apache.nlpcraft.model.{NCCustomParser, NCElement, NCModelView, NCToken, NCValue, NCValueLoader}
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
@@ -30,21 +31,18 @@
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
import java.io.{Serializable => JSerializable}
-import java.util
-import java.util.{Collections, List => JList}
-import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala}
+import java.util.{Collections, Optional, List => JList}
+import java.{lang, util}
+import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsJava, SetHasAsScala}
/**
* Probe commands processor.
*/
object NCCommandManager extends NCService {
- private final val GSON = new Gson()
+ private final val JS_MAPPER = new ObjectMapper()
- /**
- * Starts this service.
- *
- * @param parent Optional parent span.
- */
+ JS_MAPPER.registerModule(DefaultScalaModule)
+
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ =>
ackStarting()
ackStarted()
@@ -128,7 +126,7 @@
span
)
- case "S2P_MODEL_INFO" =>
+ case "S2P_MODEL_SYNS_INFO" =>
send0(
mkMsg = () => {
val mdlId = msg.data[String]("mdlId")
@@ -142,9 +140,9 @@
mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
NCProbeMessage(
- "P2S_MODEL_INFO",
+ "P2S_MODEL_SYNS_INFO",
"reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
+ "resp" -> JS_MAPPER.writeValueAsString(
Map(
"macros" -> macros.asInstanceOf[JSerializable],
"synonyms" -> syns.asInstanceOf[JSerializable],
@@ -155,7 +153,7 @@
},
mkErrorMsg = e =>
NCProbeMessage(
- "P2S_MODEL_INFO",
+ "P2S_MODEL_SYNS_INFO",
"reqGuid" -> msg.getGuid,
"error" -> e.getLocalizedMessage
),
@@ -182,7 +180,7 @@
NCProbeMessage(
"P2S_MODEL_ELEMENT_INFO",
"reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
+ "resp" -> JS_MAPPER.writeValueAsString(
Map(
"synonyms" -> elm.getSynonyms.asInstanceOf[JSerializable],
"values" -> vals.asInstanceOf[JSerializable],
@@ -200,6 +198,103 @@
span
)
+ case "S2P_MODEL_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+
+ val mdl = NCModelManager.getModel(mdlId).model
+
+ val convertedMdl =
+ new NCModelView {
+ // As is.
+ override def getId: String = mdl.getId
+ override def getName: String = mdl.getName
+ override def getVersion: String = mdl.getVersion
+ override def getDescription: String = mdl.getDescription
+ override def getOrigin: String = mdl.getOrigin
+ override def getMaxUnknownWords: Int = mdl.getMaxUnknownWords
+ override def getMaxFreeWords: Int = mdl.getMaxFreeWords
+ override def getMaxSuspiciousWords: Int = mdl.getMaxSuspiciousWords
+ override def getMinWords: Int = mdl.getMinWords
+ override def getMaxWords: Int = mdl.getMaxWords
+ override def getMinTokens: Int = mdl.getMinTokens
+ override def getMaxTokens: Int = mdl.getMaxTokens
+ override def getMinNonStopwords: Int = mdl.getMinNonStopwords
+ override def isNonEnglishAllowed: Boolean = mdl.isNonEnglishAllowed
+ override def isNotLatinCharsetAllowed: Boolean = mdl.isNotLatinCharsetAllowed
+ override def isSwearWordsAllowed: Boolean = mdl.isSwearWordsAllowed
+ override def isNoNounsAllowed: Boolean = mdl.isNoNounsAllowed
+ override def isPermutateSynonyms: Boolean = mdl.isPermutateSynonyms
+ override def isDupSynonymsAllowed: Boolean = mdl.isDupSynonymsAllowed
+ override def getMaxTotalSynonyms: Int = mdl.getMaxTotalSynonyms
+ override def isNoUserTokensAllowed: Boolean = mdl.isNoUserTokensAllowed
+ override def isSparse: Boolean = mdl.isSparse
+ override def getMetadata: util.Map[String, AnyRef] = mdl.getMetadata
+ override def getAdditionalStopWords: util.Set[String] =
+ mdl.getAdditionalStopWords
+ override def getExcludedStopWords: util.Set[String] = mdl.getExcludedStopWords
+ override def getSuspiciousWords: util.Set[String] = mdl.getSuspiciousWords
+ override def getMacros: util.Map[String, String] = mdl.getMacros
+ override def getEnabledBuiltInTokens: util.Set[String] =
+ mdl.getEnabledBuiltInTokens
+ override def getAbstractTokens: util.Set[String] = mdl.getAbstractTokens
+ override def getMaxElementSynonyms: Int = mdl.getMaxElementSynonyms
+ override def isMaxSynonymsThresholdError: Boolean =
+ mdl.isMaxSynonymsThresholdError
+ override def getConversationTimeout: Long = mdl.getConversationTimeout
+ override def getConversationDepth: Int = mdl.getConversationDepth
+ override def getRestrictedCombinations: util.Map[String, util.Set[String]] =
+ mdl.getRestrictedCombinations
+
+ // Cleared.
+ override def getParsers: JList[NCCustomParser] = null
+ // Converted.
+ override def getElements: util.Set[NCElement] = mdl.getElements.asScala.map(e =>
+ new NCElement {
+ // As is.
+ override def getId: String = e.getId
+ override def getGroups: JList[String] = e.getGroups
+ override def getMetadata: util.Map[String, AnyRef] = e.getMetadata
+ override def getDescription: String = e.getDescription
+ override def getParentId: String = e.getParentId
+ override def getSynonyms: JList[String] = e.getSynonyms
+ override def isPermutateSynonyms: Optional[lang.Boolean] =
+ e.isPermutateSynonyms
+ override def isSparse: Optional[lang.Boolean] = e.isSparse
+
+ // Cleared.
+ override def getValueLoader: Optional[NCValueLoader] = null
+ // Converted.
+ override def getValues: JList[NCValue] =
+ if (e.getValues != null) {
+ e.getValues.asScala.map(v => new NCValue {
+ override def getName: String = v.getName
+ // Cleared.
+ override def getSynonyms: JList[String] = null
+ }).asJava
+ }
+ else
+ null
+ }
+ ).asJava
+ }
+
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> JS_MAPPER.writeValueAsString(convertedMdl)
+ )
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_SYNS_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
+ span
+ )
+
case _ =>
logger.error(s"Received unknown server message (you need to update the probe): ${msg.getType}")
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 04792f6..3c7c46c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -25,7 +25,7 @@
import org.apache.nlpcraft.common.crypto.NCCipher
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.NCNlpSentence
-import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreManager, NCNlpPorterStemmer}
+import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.pool.NCThreadPoolManager
import org.apache.nlpcraft.common.socket.NCSocket
import org.apache.nlpcraft.common.version.NCVersion
@@ -54,8 +54,7 @@
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private final val TYPE_MODEL_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
- private final val TYPE_MODEL_ELEMENT_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
+ private final val TYPE_JAVA_META = new TypeToken[JavaMeta]() {}.getType
// Type safe and eager configuration container.
private object Config extends NCConfigurable {
@@ -67,7 +66,7 @@
def reconnectTimeoutMs: Long = getLong(s"$pre.reconnectTimeoutMs")
def pingTimeoutMs: Long = getLong(s"$pre.pingTimeoutMs")
def soTimeoutMs: Int = getInt(s"$pre.soTimeoutMs")
-
+
/**
*
*/
@@ -158,8 +157,9 @@
// All probes pending complete handshake keyed by probe key.
@volatile private var pending: mutable.Map[ProbeKey, ProbeHolder] = _
- @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelsSynsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
@volatile private var modelElmsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
/**
*
@@ -181,8 +181,9 @@
"downlink" -> s"$dnHost:$dnPort"
)
- modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelsSynsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
modelElmsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
dnSrv = startServer("Downlink", dnHost, dnPort, downLinkHandler)
upSrv = startServer("Uplink", upHost, upPort, upLinkHandler)
@@ -219,8 +220,9 @@
U.stopThread(dnSrv)
U.stopThread(upSrv)
- modelsInfo = null
+ modelsSynsInfo = null
modelElmsInfo = null
+ modelsInfo = null
ackStopped()
}
@@ -691,6 +693,23 @@
}
}
}
+
+ /**
+ *
+ * @param probeMsg
+ * @param m
+ */
+ private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = {
+ val p = m.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
+ else
+ logger.warn(s"Message ignored: $probeMsg")
+ }
/**
* Processes the messages received from the probe.
@@ -716,27 +735,9 @@
typ match {
case "P2S_PING" => ()
- case "P2S_MODEL_INFO" =>
- val p = modelsInfo.remove(probeMsg.data[String]("reqGuid"))
-
- if (p != null)
- probeMsg.dataOpt[String]("resp") match {
- case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_INFO_RESP))
- case None => p.failure(new NCE(probeMsg.data[String]("error")))
- }
- else
- logger.warn(s"Message ignored: $probeMsg")
-
- case "P2S_MODEL_ELEMENT_INFO" =>
- val p = modelElmsInfo.remove(probeMsg.data[String]("reqGuid"))
-
- if (p != null)
- probeMsg.dataOpt[String]("resp") match {
- case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_ELEMENT_INFO_RESP))
- case None => p.failure(new NCE(probeMsg.data[String]("error")))
- }
- else
- logger.warn(s"Message ignored: $probeMsg")
+ case "P2S_MODEL_SYNS_INFO" => processJavaMetaMessage(probeMsg, modelsSynsInfo)
+ case "P2S_MODEL_ELEMENT_INFO" => processJavaMetaMessage(probeMsg, modelElmsInfo)
+ case "P2S_MODEL_INFO" => processJavaMetaMessage(probeMsg, modelsInfo)
case "P2S_ASK_RESULT" =>
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -1089,12 +1090,12 @@
* @param parent
* @return
*/
- def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
- startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
+ def getModelSynonymsInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelSynonymsInfo", parent, "mdlId" -> mdlId) { _ =>
processModelDataRequest(
mdlId,
- NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
- modelsInfo,
+ NCProbeMessage("S2P_MODEL_SYNS_INFO", "mdlId" -> mdlId),
+ modelsSynsInfo,
parent
)
}
@@ -1142,4 +1143,15 @@
}
)
}
+
+ def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
+ modelsInfo,
+ parent
+ )
+ }
+
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index e74fb68..39b84e4 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -879,6 +879,51 @@
}
}
+
+ /**
+ *
+ * @return
+ */
+ protected def $model$info(): Route = {
+ case class Req$Model$Info(
+ acsTok: String,
+ mdlId: String
+ )
+
+ implicit val reqFmt: RootJsonFormat[Req$Model$Info] = jsonFormat2(Req$Model$Info)
+
+ entity(as[Req$Model$Info]) { req =>
+ startScopedSpan(
+ "model$syns",
+ "acsTok" -> req.acsTok,
+ "mdlId" -> req.mdlId
+ ) { span =>
+ checkLength("acsTok" -> req.acsTok, "mdlId" -> req.mdlId)
+
+ val admUsr = authenticateAsAdmin(req.acsTok)
+ val compId = admUsr.companyId
+
+ if (!NCProbeManager.existsForModel(compId, req.mdlId))
+ throw InvalidModelId(req.mdlId)
+
+ val fut = NCProbeManager.getModelInfo(req.mdlId, span)
+
+ successWithJs(
+ fut.collect {
+ // We have to use Jackson (not spray) here to serialize 'result' field.
+ case res =>
+ toJs(
+ Map(
+ "status" -> API_OK.toString,
+ "model" -> res
+ )
+ )
+ }
+ )
+ }
+ }
+ }
+
/**
*
* @return
@@ -2040,6 +2085,7 @@
path(API / "probe" / "all") { withMetric(M_PROBE_ALL_LATENCY_MS, $probe$All) } ~
path(API / "model" / "sugsyn") { withMetric(M_MODEL_SUGSYN_LATENCY_MS, $model$sugsyn) } ~
path(API / "model" / "syns") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$syns) } ~
+ path(API / "model" / "info") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$info) } ~
path(API / "ask") { withMetric(M_ASK_LATENCY_MS, $ask) } ~
path(API / "ask" / "sync") { withMetric(M_ASK_SYNC_LATENCY_MS, $ask$Sync) }
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index fae55d0..d89ba98 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -143,7 +143,7 @@
val promise = Promise[NCSuggestSynonymResult]()
- NCProbeManager.getModelInfo(mdlId, parent).onComplete {
+ NCProbeManager.getModelSynonymsInfo(mdlId, parent).onComplete {
case Success(m) =>
try {
require(
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index 0b8c03f..d7d22b8 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -111,4 +111,15 @@
postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> ("A" * 65))
postError("model/syns", 400, "NC_ERROR", "mdlId" -> "rest.test.model")
}
+
+ @Test
+ def testModelInfo(): Unit = {
+ post("model/info", "mdlId" -> "rest.test.model")(
+ ("$.status", (status: String) => assertEquals("API_OK", status)),
+ ("$.model", (data: java.util.Map[Object, Object]) => assertTrue(!data.isEmpty))
+ )
+
+ postError("model/info", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN")
+ postError("model/info", 400, "NC_ERROR")
+ }
}
