Merge branch 'master' into NLPCRAFT-369
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 4503970..a1c8f97 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -1667,6 +1667,10 @@
else
try
GSON.toJson(GSON.getAdapter(classOf[JsonElement]).fromJson(json))
+ // Fix the problem with escaping '<' and '>' which is only
+ // a theoretical problem for browsers displaying JSON.
+ .replace("\\u003c", "<")
+ .replace("\\u003e", ">")
catch {
case _: Exception => ""
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
index 4a6e3a5..edd9561 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
@@ -159,6 +159,11 @@
int MODEL_VERSION_MAXLEN = 16;
/**
+ * Max length for {@link NCElement#getId()} method.
+ */
+ int MODEL_ELEMENT_ID_MAXLEN = 64;
+
+ /**
* Default value for {@link #isSparse()} method.
*/
boolean DFLT_IS_SPARSE = false;
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
index 401ec67..549372d 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
@@ -147,7 +147,9 @@
private final val REST_CMD = CMDS.find(_.name == "rest").get
private final val CALL_CMD = CMDS.find(_.name == "call").get
private final val ASK_CMD = CMDS.find(_.name == "ask").get
- private final val SUGSYN_CMD = CMDS.find(_.name == "sugsyn").get
+ private final val MODEL_SUGSYN_CMD = CMDS.find(_.name == "model-sugsyn").get
+ private final val MODEL_SYNS_CMD = CMDS.find(_.name == "model-syns").get
+ private final val MODEL_INFO_CMD = CMDS.find(_.name == "model-info").get
/**
* @param cmd
@@ -549,6 +551,7 @@
logln(r(" [Error]"))
error(s"Server start failed - check full log for errors: ${c(output.getAbsolutePath)}")
+ error(s"If the problem persists - remove ${c("~/.nlpcraft")} folder and try again.")
tailFile(output.getAbsolutePath, 20)
}
@@ -1555,6 +1558,7 @@
*/
private [cmdline] def cmdInfo(cmd: Command, args: Seq[Argument], repl: Boolean): Unit = {
cmdInfoServer(CMDS.find(_.name == "info-server").get, Seq.empty[Argument], repl)
+ logln()
cmdInfoProbe(CMDS.find(_.name == "info-probe").get, Seq.empty[Argument], repl)
}
@@ -1737,7 +1741,7 @@
* @param args Arguments, if any, for this command.
* @param repl Whether or not executing from REPL.
*/
- private [cmdline] def cmdSugSyn(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ private [cmdline] def cmdModelSugSyn(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
state.accessToken match {
case Some(acsTok) =>
val mdlId = getParamOpt(args, "mdlId") match {
@@ -1771,6 +1775,72 @@
* @param args Arguments, if any, for this command.
* @param repl Whether or not executing from REPL.
*/
+ private [cmdline] def cmdModelSyns(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ state.accessToken match {
+ case Some(acsTok) =>
+ val mdlId = getParamOpt(args, "mdlId") match {
+ case Some(id) => id
+ case None =>
+ if (state.probes.size == 1 && state.probes.head.models.length == 1)
+ state.probes.head.models.head.id
+ else
+ throw MissingOptionalParameter(cmd, "mdlId")
+ }
+ val elmId = getParam(cmd, args, "elmId")
+
+ httpRest(
+ cmd,
+ "model/syns",
+ s"""
+ |{
+ | "acsTok": ${jsonQuote(acsTok)},
+ | "mdlId": ${jsonQuote(mdlId)},
+ | "elmId": ${jsonQuote(elmId)}
+ |}
+ |""".stripMargin
+ )
+
+ case None => throw NotSignedIn()
+ }
+
+ /**
+ *
+ * @param cmd Command descriptor.
+ * @param args Arguments, if any, for this command.
+ * @param repl Whether or not executing from REPL.
+ */
+ private [cmdline] def cmdModelInfo(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ state.accessToken match {
+ case Some(acsTok) =>
+ val mdlId = getParamOpt(args, "mdlId") match {
+ case Some(id) => id
+ case None =>
+ if (state.probes.size == 1 && state.probes.head.models.length == 1)
+ state.probes.head.models.head.id
+ else
+ throw MissingOptionalParameter(cmd, "mdlId")
+ }
+
+ httpRest(
+ cmd,
+ "model/info",
+ s"""
+ |{
+ | "acsTok": ${jsonQuote(acsTok)},
+ | "mdlId": ${jsonQuote(mdlId)}
+ |}
+ |""".stripMargin
+ )
+
+ case None => throw NotSignedIn()
+ }
+
+ /**
+ *
+ * @param cmd Command descriptor.
+ * @param args Arguments, if any, for this command.
+ * @param repl Whether or not executing from REPL.
+ */
private [cmdline] def cmdAsk(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
state.accessToken match {
case Some(acsTok) =>
@@ -2520,7 +2590,7 @@
mkCandidate(
disp = name,
- grp = s"REST ${cmd.group}:",
+ grp = MANDATORY_GRP,
desc = cmd.desc,
completed = true
)
@@ -2529,20 +2599,50 @@
)
}
- // For 'ask' and 'sugysn' - add additional model IDs auto-completion/suggestion candidates.
- if (cmd == ASK_CMD.name || cmd == SUGSYN_CMD.name)
+ // For 'ask', 'sugsyn', 'model-syn' and 'model-info' - add additional
+ // model IDs auto-completion/suggestion candidates.
+ if (cmd == ASK_CMD.name || cmd == MODEL_SUGSYN_CMD.name || cmd == MODEL_SYNS_CMD.name || cmd == MODEL_INFO_CMD.name)
candidates.addAll(
- state.probes.flatMap(_.models.toList).map(mdl => {
+ state.probes.flatMap(_.models.toList).map(mdl =>
mkCandidate(
disp = s"--mdlId=${mdl.id}",
grp = MANDATORY_GRP,
desc = null,
completed = true
)
- })
+ )
.asJava
)
+ // For 'model-syns' add auto-completion for element IDs.
+ if (cmd == MODEL_SYNS_CMD.name) {
+ val mdlIdParam = MODEL_SYNS_CMD.findParameterById("mdlId")
+
+ words.find(w => mdlIdParam.names.exists(x => w.startsWith(x))) match {
+ case Some(p) =>
+ val mdlId = p.substring(p.indexOf('=') + 1)
+
+ state.probes.flatMap(_.models).find(_.id == mdlId) match {
+ case Some(mdl) =>
+ candidates.addAll(
+ mdl.elementIds.toList.map(id =>
+ mkCandidate(
+ disp = s"--elmId=$id",
+ grp = MANDATORY_GRP,
+ desc = null,
+ completed = true
+ )
+ )
+ .asJava
+ )
+
+ case None => ()
+ }
+
+ case None => ()
+ }
+ }
+
// For 'call' - add additional auto-completion/suggestion candidates.
if (cmd == CALL_CMD.name) {
val pathParam = CALL_CMD.findParameterById("path")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliBase.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliBase.scala
index 96c219c..0bd7531 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliBase.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliBase.scala
@@ -121,10 +121,11 @@
// See NCProbeModelMdo.
case class ProbeModel(
- id: String,
- name: String,
- version: String,
- enabledBuiltInTokens: Array[String]
+ id: String,
+ name: String,
+ version: String,
+ elementIds: Array[String],
+ enabledBuiltInTokens: Array[String]
)
case class ProbeAllResponse(
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliCommands.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliCommands.scala
index 0f86e13..3b43ef0 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliCommands.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliCommands.scala
@@ -95,6 +95,7 @@
private final lazy val SCRIPT_NAME = U.sysEnv("NLPCRAFT_CLI_SCRIPT").getOrElse(s"nlpcraft.${if (SystemUtils.IS_OS_UNIX) "sh" else "cmd"}")
private final lazy val PROMPT = if (SCRIPT_NAME.endsWith("cmd")) ">" else "$"
private final lazy val VER = NCVersion.getCurrent
+ private final lazy val REST_SPEC_URL = "https://nlpcraft.apache.org/using-rest.html"
//noinspection DuplicatedCode
// All supported commands.
@@ -104,7 +105,7 @@
group = "2. REST Commands",
synopsis = s"REST call in a convenient way for command line mode.",
desc = Some(
- s"When using this command you supply all call parameters as a single ${c("'--json'")} parameter with a JSON string. " +
+ s"When using this command you supply all call parameters as a single ${y("'--json'")} parameter with a JSON string. " +
s"In REPL mode, you can hit ${rv(" Tab ")} to see auto-suggestion and auto-completion candidates for " +
s"commonly used paths. However, ${y("'call'")} command provides more convenient way to issue REST " +
s"calls when in REPL mode."
@@ -158,6 +159,7 @@
group = "2. REST Commands",
synopsis = s"Wrapper for ${y("'/signin'")} REST call.",
desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
s"If no arguments provided, it signs in with the " +
s"default 'admin@admin.com' user account. NOTE: please make sure to remove this account when " +
s"running in production."
@@ -195,6 +197,7 @@
group = "2. REST Commands",
synopsis = s"Wrapper for ${y("'/signout'")} REST call in REPL mode.",
desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
s"Signs out currently signed in user. Note that this command makes sense only in REPL mode."
),
body = NCCli.cmdSignOut,
@@ -246,18 +249,18 @@
examples = Seq(
Example(
usage = Seq(
- s"$PROMPT $SCRIPT_NAME call -p=signin",
+ s"> call -p=signin",
" --email=admin@admin.com",
" --passwd=admin"
),
desc =
s"Issues ${y("'signin'")} REST call with given JSON payload provided as a set of parameters. " +
- s"Note that ${c("'--email'")} and ${c("'--passwd'")} parameters correspond to the REST call " +
+ s"Note that ${y("'--email'")} and ${y("'--passwd'")} parameters correspond to the REST call " +
s"specification for ${y("'/signin'")} path."
),
Example(
usage = Seq(
- s"$$ nlpcraft.sh call --path=ask/sync",
+ s"> call --path=ask/sync",
" --acsTok=qwerty123456",
" --txt=\"User request\"",
" --mdlId=my.model.id",
@@ -265,28 +268,16 @@
" --enableLog=false"
),
desc =
- s"${bo("Unix/Linux:")} issues ${y("'ask/sync'")} REST call with given JSON payload provided as a set of parameters."
- ),
- Example(
- usage = Seq(
- s"> nlpcraft.cmd call --path=ask/sync",
- " --acsTok=qwerty123456",
- " --txt=\"User request\"",
- " --mdlId=my.model.id",
- " --data='{\\\"data1\\\": true, \\\"data2\\\": 123, \\\"data3\\\": \\\"some text\\\"}'",
- " --enableLog=false"
- ),
- desc =
- s"${bo("Windows:")} issues ${y("'ask/sync'")} REST call with given JSON payload provided " +
- s"as a set of parameters. Note the necessary double quote escaping."
+ s"Issues ${y("'ask/sync'")} REST call with given JSON payload provided as a set of parameters."
)
)
),
Command(
name = "ask",
group = "2. REST Commands",
- synopsis = s"Wrapper for ${c("'/ask/sync'")} REST call.",
+ synopsis = s"Wrapper for ${y("'/ask/sync'")} REST call.",
desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
s"Requires user to be already signed in. This command ${bo("only makes sense in the REPL mode")} as " +
s"it requires user to be signed in. REPL session keeps the currently active access " +
s"token after user signed in. For command line mode, use ${y("'rest'")} command with " +
@@ -369,7 +360,7 @@
value = Some("class"),
desc =
s"Mandatory JDBC driver class. Note that 'class' must be a fully qualified class name. " +
- s"It should also be available on the classpath or provided additionally via ${c("'--cp'")} parameter."
+ s"It should also be available on the classpath or provided additionally via ${y("'--cp'")} parameter."
),
Parameter(
id = "schema",
@@ -397,9 +388,9 @@
s"Additional JVM classpath that will be appended to the default NLPCraft JVM classpath. " +
s"Parameter should include one or more classpath entry (JAR or directory) separated by the OS specific classpath separator. " +
s"Although this configuration property is optional, in most cases you will need to provide an " +
- s"additional classpath for JDBC driver that you use (see ${c("'--driver'")} parameter) unless " +
+ s"additional classpath for JDBC driver that you use (see ${y("'--driver'")} parameter) unless " +
s"it is available in NLPCraft by default, i.e. Apache Ignite and H2. " +
- s"Note that you can have multiple ${c("'--cp'")} parameters and their values will be " +
+ s"Note that you can have multiple ${y("'--cp'")} parameters and their values will be " +
s"automatically combined into one final additional classpath. " +
s"Note also that you can use ${y("'~'")} at the beginning of the classpath component to specify user home directory."
),
@@ -454,7 +445,7 @@
optional = true,
desc =
s"Semicolon-separate list of tables and/or columns to exclude. By default, none of the " +
- s"tables and columns in the schema are excluded. See ${c("'--help'")} parameter to get more details."
+ s"tables and columns in the schema are excluded. See ${y("'--help'")} parameter to get more details."
),
Parameter(
id = "include",
@@ -463,7 +454,7 @@
optional = true,
desc =
s"Semicolon-separate list of tables and/or columns to include. By default, all of the " +
- s"tables and columns in the schema are included. See ${c("'--help'")} parameter to get more details."
+ s"tables and columns in the schema are included. See ${y("'--help'")} parameter to get more details."
),
Parameter(
id = "prefix",
@@ -555,10 +546,91 @@
)
),
Command(
- name = "sugsyn",
+ name = "model-info",
+ group = "2. REST Commands",
+ synopsis = s"Wrapper for ${y("'/model/info'")} REST call.",
+ desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
+ s"Requires user to be already signed in. This command ${bo("only makes sense in the REPL mode")} as " +
+ s"it requires user to be signed in. REPL session keeps the currently active access " +
+ s"token after user signed in. For command line mode, use ${y("'rest'")} command with " +
+ s"corresponding parameters. Note also that it requires a local probe running that hosts " +
+ s"the specified model."
+ ),
+ body = NCCli.cmdModelInfo,
+ params = Seq(
+ Parameter(
+ id = "mdlId",
+ names = Seq("--mdlId", "-m"),
+ value = Some("model.id"),
+ optional = true,
+ desc =
+ s"ID of the model to get configuration properties and elements for. " +
+ s"In REPL mode, hit ${rv(" Tab ")} to see auto-suggestion for possible model IDs. Note that " +
+ s"this is optional ONLY if there is only one connected probe and it has only one model deployed - which will be " +
+ s"used by default. In all other cases - this parameter is mandatory."
+ )
+ ),
+ examples = Seq(
+ Example(
+ usage = Seq(
+ s"""> model-info -m=my.model.id"""
+ ),
+ desc =
+ s"Issues ${y("'model/info'")} REST call with given model ID."
+ )
+ )
+ ),
+ Command(
+ name = "model-syns",
+ group = "2. REST Commands",
+ synopsis = s"Wrapper for ${y("'/model/syns'")} REST call.",
+ desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
+ s"Requires user to be already signed in. This command ${bo("only makes sense in the REPL mode")} as " +
+ s"it requires user to be signed in. REPL session keeps the currently active access " +
+ s"token after user signed in. For command line mode, use ${y("'rest'")} command with " +
+ s"corresponding parameters. Note also that it requires a local probe running that hosts " +
+ s"the specified model."
+ ),
+ body = NCCli.cmdModelSyns,
+ params = Seq(
+ Parameter(
+ id = "mdlId",
+ names = Seq("--mdlId", "-m"),
+ value = Some("model.id"),
+ optional = true,
+ desc =
+ s"ID of the model to get expanded synonyms for. " +
+ s"In REPL mode, hit ${rv(" Tab ")} to see auto-suggestion for possible model IDs. Note that " +
+ s"this is optional ONLY if there is only one connected probe and it has only one model deployed - which will be " +
+ s"used by default. In all other cases - this parameter is mandatory."
+ ),
+ Parameter(
+ id = "elmId",
+ names = Seq("--elmId", "-e"),
+ value = Some("element.id"),
+ desc =
+ s"ID of the model element to get expanded synonyms and values for. " +
+ s"In REPL mode, hit ${rv(" Tab ")} to see auto-suggestion for possible element IDs."
+ )
+ ),
+ examples = Seq(
+ Example(
+ usage = Seq(
+ s"""> model-syns -m=my.model.id -e=my:elem"""
+ ),
+ desc =
+ s"Issues ${y("'model/syns'")} REST call with given model and element IDs."
+ )
+ )
+ ),
+ Command(
+ name = "model-sugsyn",
group = "2. REST Commands",
synopsis = s"Wrapper for ${y("'/model/sugsyn'")} REST call.",
desc = Some(
+ s"See ${REST_SPEC_URL} for REST call specification. " +
s"Requires user to be already signed in. This command ${bo("only makes sense in the REPL mode")} as " +
s"it requires user to be signed in. REPL session keeps the currently active access " +
s"token after user signed in. For command line mode, use ${y("'rest'")} command with " +
@@ -566,7 +638,7 @@
s"the specified model as well as running ${y("'ctxword'")} server. Find more information about " +
s"this tool at https://nlpcraft.apache.org/tools/syn_tool.html"
),
- body = NCCli.cmdSugSyn,
+ body = NCCli.cmdModelSugSyn,
params = Seq(
Parameter(
id = "mdlId",
@@ -590,14 +662,14 @@
examples = Seq(
Example(
usage = Seq(
- s"""> sugsyn -m=my.model.id"""
+ s"""> model-sugsyn -m=my.model.id"""
),
desc =
s"Issues ${y("'model/sugsyn'")} REST call with default min score and given model ID."
),
Example(
usage = Seq(
- s"""> sugsyn"""
+ s"""> model-sugsyn"""
),
desc =
s"Issues ${y("'model/sugsyn'")} REST call with default min score and default model ID " +
@@ -659,7 +731,7 @@
synopsis = s"Starts local server.",
desc = Some(
s"Server is started in the external JVM process with both stdout and stderr piped out into log file. " +
- s"Command will block until the server is started unless ${c("'--noWait'")} parameter is used or timeout is expired."
+ s"Command will block until the server is started unless ${y("'--noWait'")} parameter is used or timeout is expired."
),
body = NCCli.cmdStartServer,
params = Seq(
@@ -732,7 +804,7 @@
synopsis = s"Starts local probe.",
desc = Some(
s"Probe is started in the external JVM process with both stdout and stderr piped out into log file. " +
- s"Command will block until the probe is started unless ${c("'--noWait'")} parameter is used or timeout is expired."
+ s"Command will block until the probe is started unless ${y("'--noWait'")} parameter is used or timeout is expired."
),
body = NCCli.cmdStartProbe,
params = Seq(
@@ -746,7 +818,7 @@
s"When starting a probe with your models you must " +
s"provide this additional classpath for the models and their dependencies this probe will be hosting. " +
s"Parameter should include one or more classpath entry (JAR or directory) separated by the OS specific classpath separator. " +
- s"Note that you can have multiple ${c("'--cp'")} parameters and their values will be " +
+ s"Note that you can have multiple ${y("'--cp'")} parameters and their values will be " +
s"automatically combined into one final additional classpath. " +
s"Note also that you can use ${y("'~'")} at the beginning of the classpath component to specify user home directory."
),
@@ -771,8 +843,8 @@
desc =
s"Comma separated list of fully qualified class names for models to deploy. This will override " +
s"${y("'nlpcraft.probe.models'")} configuration property from either default configuration file " +
- s"or the one provided by ${c("'--cfg'")} parameter. Note that you also must provide the additional " +
- s"classpath in this case via ${c("'--cp'")} parameter. Note also that you can have multiple '${c("'--mdls'")} " +
+ s"or the one provided by ${y("'--cfg'")} parameter. Note that you also must provide the additional " +
+ s"classpath in this case via ${y("'--cp'")} parameter. Note also that you can have multiple '${y("'--mdls'")} " +
s"parameters - each specifying one or more model class names - and they will be automatically combined together."
),
Parameter(
@@ -826,7 +898,7 @@
synopsis = s"Restarts local probe (REPL mode only).",
desc = Some(
s"Restart local probe with same parameters as the last start. Works only in REPL mode and only if local " +
- s"probe was previously started using ${c("'start-probe")} command. This command provides a " +
+ s"probe was previously started using ${y("'start-probe")} command. This command provides a " +
s"convenient way to quickly restart the probe to reload the model during model development and testing."
),
body = NCCli.cmdRestartProbe,
@@ -860,7 +932,7 @@
s"Additional JVM classpath that will be appended to the default NLPCraft JVM classpath. " +
s"When testing your models you must provide this additional classpath for the models and their dependencies. " +
s"Parameter should include one or more classpath entry (JAR or directory) separated by the OS specific classpath separator. " +
- s"Note that you can have multiple ${c("'--cp'")} parameters and their values will be " +
+ s"Note that you can have multiple ${y("'--cp'")} parameters and their values will be " +
s"automatically combined into one final additional classpath. " +
s"Note also that you can use ${y("'~'")} at the beginning of the classpath component to specify user home directory."
),
@@ -883,9 +955,9 @@
optional = true,
desc =
s"Comma separated list of fully qualified class names for models to deploy and test. Note that you also " +
- s"must provide the additional classpath via ${c("'--cp'")} parameter. If not provided, the models " +
- s"specified in configuration file (${c("'--cfg'")} parameter) will be used instead. Note that " +
- s"you can have multiple ${c("'--mdls'")} parameters - each specifying one or more model class " +
+ s"must provide the additional classpath via ${y("'--cp'")} parameter. If not provided, the models " +
+ s"specified in configuration file (${y("'--cfg'")} parameter) will be used instead. Note that " +
+ s"you can have multiple ${y("'--mdls'")} parameters - each specifying one or more model class " +
s"names - and they will be automatically combined together."
),
Parameter(
@@ -1074,7 +1146,7 @@
group = "3. Miscellaneous",
synopsis = s"Displays help for ${y(s"'$SCRIPT_NAME'")}.",
desc = Some(
- s"By default, without ${c("'--all'")} or ${c("'--cmd'")} parameters, displays the abbreviated form of manual " +
+ s"By default, without ${y("'--all'")} or ${y("'--cmd'")} parameters, displays the abbreviated form of manual " +
s"only listing the commands without parameters or examples."
),
body = NCCli.cmdHelp,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliRestSpec.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliRestSpec.scala
index 4c23223..7bbdc61 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliRestSpec.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliRestSpec.scala
@@ -78,6 +78,25 @@
)
),
RestSpec(
+ "model/syns",
+ "Gets model element synonyms",
+ "Tools",
+ params = Seq(
+ RestSpecParameter(name = "acsTok", kind = STRING),
+ RestSpecParameter(name = "mdlId", kind = STRING),
+ RestSpecParameter(name = "elmId", kind = STRING)
+ )
+ ),
+ RestSpec(
+ "model/info",
+ "Gets model configuration and elements",
+ "Tools",
+ params = Seq(
+ RestSpecParameter(name = "acsTok", kind = STRING),
+ RestSpecParameter(name = "mdlId", kind = STRING)
+ )
+ ),
+ RestSpec(
"check",
"Gets status and result of submitted requests",
"Asking",
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 1f96350..4c89d54 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -17,12 +17,12 @@
package org.apache.nlpcraft.probe.mgrs.cmd
-import java.io.{Serializable => JSerializable}
-import com.google.gson.Gson
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.{NCService, _}
import org.apache.nlpcraft.common.nlp.NCNlpSentence
-import org.apache.nlpcraft.model.NCToken
+import org.apache.nlpcraft.common.{NCService, _}
+import org.apache.nlpcraft.model.{NCCustomParser, NCElement, NCModelView, NCToken, NCValue, NCValueLoader}
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
@@ -30,22 +30,19 @@
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
-import java.util
-import java.util.{List => JList}
-
-import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala}
+import java.io.{Serializable => JSerializable}
+import java.util.{Collections, Optional, List => JList}
+import java.{lang, util}
+import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsJava, SetHasAsScala}
/**
* Probe commands processor.
*/
object NCCommandManager extends NCService {
- private final val GSON = new Gson()
+ private final val JS_MAPPER = new ObjectMapper()
- /**
- * Starts this service.
- *
- * @param parent Optional parent span.
- */
+ JS_MAPPER.registerModule(DefaultScalaModule)
+
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ =>
ackStarting()
ackStarted()
@@ -63,6 +60,28 @@
/**
*
+ * @param mkMsg
+ * @param mkErrorMsg
+ */
+ private def send0(mkMsg: () => NCProbeMessage, mkErrorMsg: Throwable => NCProbeMessage, parent: Span = null): Unit = {
+ val msgOpt: Option[NCProbeMessage] =
+ try
+ Some(mkMsg())
+ catch {
+ case e: Throwable =>
+ NCConnectionManager.send(mkErrorMsg(e), parent)
+
+ None
+ }
+
+ msgOpt match {
+ case Some(msg) => NCConnectionManager.send(msg, parent)
+ case None => // No-op.
+ }
+ }
+
+ /**
+ *
* @param msg Server message to process.
* @param parent Optional parent span.
*/
@@ -107,27 +126,172 @@
span
)
- case "S2P_MODEL_INFO" =>
- val mdlId = msg.data[String]("mdlId")
+ case "S2P_MODEL_SYNS_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
- val mdlData = NCModelManager.getModel(mdlId)
+ val mdlData = NCModelManager.getModel(mdlId)
- val macros: util.Map[String, String] = mdlData.model.getMacros
- val syns: util.Map[String, util.List[String]] = mdlData.model.getElements.asScala.map(p => p.getId -> p.getSynonyms).toMap.asJava
- val samples: util.Map[String, util.List[util.List[String]]] = mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
+ val macros: util.Map[String, String] = mdlData.model.getMacros
+ val syns: util.Map[String, util.List[String]] =
+ mdlData.model.getElements.asScala.map(p => p.getId -> p.getSynonyms).toMap.asJava
+ val samples: util.Map[String, util.List[util.List[String]]] =
+ mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
- NCConnectionManager.send(
- NCProbeMessage(
- "P2S_MODEL_INFO",
- "reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
- Map(
- "macros" -> macros.asInstanceOf[JSerializable],
- "synonyms" -> syns.asInstanceOf[JSerializable],
- "samples" -> samples.asInstanceOf[JSerializable]
- ).asJava
+ NCProbeMessage(
+ "P2S_MODEL_SYNS_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> JS_MAPPER.writeValueAsString(
+ Map(
+ "macros" -> macros.asInstanceOf[JSerializable],
+ "synonyms" -> syns.asInstanceOf[JSerializable],
+ "samples" -> samples.asInstanceOf[JSerializable]
+ ).asJava
+ )
)
- ),
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_SYNS_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
+ span
+ )
+
+ case "S2P_MODEL_ELEMENT_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+ val elmId = msg.data[String]("elmId")
+
+ val mdl = NCModelManager.getModel(mdlId).model
+
+ val elm = mdl.getElements.asScala.find(_.getId == elmId).
+ getOrElse(throw new NCE(s"Element not found in model: $elmId"))
+
+ val vals: util.Map[String, JList[String]] =
+ if (elm.getValues != null)
+ elm.getValues.asScala.map(e => e.getName -> e.getSynonyms).toMap.asJava
+ else
+ Collections.emptyMap()
+
+ NCProbeMessage(
+ "P2S_MODEL_ELEMENT_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> JS_MAPPER.writeValueAsString(
+ Map(
+ "synonyms" -> elm.getSynonyms.asInstanceOf[JSerializable],
+ "values" -> vals.asInstanceOf[JSerializable],
+ "macros" -> mdl.getMacros.asInstanceOf[JSerializable]
+ ).asJava
+ )
+ )
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_ELEMENT_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
+ span
+ )
+
+ case "S2P_MODEL_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+
+ val mdl = NCModelManager.getModel(mdlId).model
+
+ val convertedMdl =
+ new NCModelView {
+ // As is.
+ override def getId: String = mdl.getId
+ override def getName: String = mdl.getName
+ override def getVersion: String = mdl.getVersion
+ override def getDescription: String = mdl.getDescription
+ override def getOrigin: String = mdl.getOrigin
+ override def getMaxUnknownWords: Int = mdl.getMaxUnknownWords
+ override def getMaxFreeWords: Int = mdl.getMaxFreeWords
+ override def getMaxSuspiciousWords: Int = mdl.getMaxSuspiciousWords
+ override def getMinWords: Int = mdl.getMinWords
+ override def getMaxWords: Int = mdl.getMaxWords
+ override def getMinTokens: Int = mdl.getMinTokens
+ override def getMaxTokens: Int = mdl.getMaxTokens
+ override def getMinNonStopwords: Int = mdl.getMinNonStopwords
+ override def isNonEnglishAllowed: Boolean = mdl.isNonEnglishAllowed
+ override def isNotLatinCharsetAllowed: Boolean = mdl.isNotLatinCharsetAllowed
+ override def isSwearWordsAllowed: Boolean = mdl.isSwearWordsAllowed
+ override def isNoNounsAllowed: Boolean = mdl.isNoNounsAllowed
+ override def isPermutateSynonyms: Boolean = mdl.isPermutateSynonyms
+ override def isDupSynonymsAllowed: Boolean = mdl.isDupSynonymsAllowed
+ override def getMaxTotalSynonyms: Int = mdl.getMaxTotalSynonyms
+ override def isNoUserTokensAllowed: Boolean = mdl.isNoUserTokensAllowed
+ override def isSparse: Boolean = mdl.isSparse
+ override def getMetadata: util.Map[String, AnyRef] = mdl.getMetadata
+ override def getAdditionalStopWords: util.Set[String] = mdl.getAdditionalStopWords
+ override def getExcludedStopWords: util.Set[String] = mdl.getExcludedStopWords
+ override def getSuspiciousWords: util.Set[String] = mdl.getSuspiciousWords
+ override def getMacros: util.Map[String, String] = mdl.getMacros
+ override def getEnabledBuiltInTokens: util.Set[String] = mdl.getEnabledBuiltInTokens
+ override def getAbstractTokens: util.Set[String] = mdl.getAbstractTokens
+ override def getMaxElementSynonyms: Int = mdl.getMaxElementSynonyms
+ override def isMaxSynonymsThresholdError: Boolean = mdl.isMaxSynonymsThresholdError
+ override def getConversationTimeout: Long = mdl.getConversationTimeout
+ override def getConversationDepth: Int = mdl.getConversationDepth
+ override def getRestrictedCombinations: util.Map[String, util.Set[String]] = mdl.getRestrictedCombinations
+
+ // Cleared.
+ override def getParsers: JList[NCCustomParser] = null
+
+ // Add list of just element IDs.
+ val elementIds: JList[String] = mdl.getElements.asScala.map(_.getId).toList.asJava
+
+ // Converted.
+ override def getElements: util.Set[NCElement] = mdl.getElements.asScala.map(e =>
+ new NCElement {
+ // As is.
+ override def getId: String = e.getId
+ override def getGroups: JList[String] = e.getGroups
+ override def getMetadata: util.Map[String, AnyRef] = e.getMetadata
+ override def getDescription: String = e.getDescription
+ override def getParentId: String = e.getParentId
+ override def getSynonyms: JList[String] = e.getSynonyms
+ override def isPermutateSynonyms: Optional[lang.Boolean] = e.isPermutateSynonyms
+ override def isSparse: Optional[lang.Boolean] = e.isSparse
+
+ // Cleared.
+ override def getValueLoader: Optional[NCValueLoader] = null
+
+ // Converted.
+ override def getValues: JList[NCValue] =
+ if (e.getValues != null) {
+ e.getValues.asScala.map(v => new NCValue {
+ override def getName: String = v.getName
+ // Cleared.
+ override def getSynonyms: JList[String] = null
+ }).asJava
+ }
+ else
+ null
+ }
+ ).asJava
+ }
+
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> JS_MAPPER.writeValueAsString(convertedMdl)
+ )
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_SYNS_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
span
)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
index 362e930..d2a4619 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
@@ -34,6 +34,7 @@
import java.util.concurrent.CountDownLatch
import java.util.{Properties, TimeZone}
import scala.collection.mutable
+import scala.jdk.CollectionConverters.{SetHasAsJava, SetHasAsScala}
/**
* Probe down/up link connection manager.
@@ -221,7 +222,8 @@
mdl.getId,
mdl.getName,
mdl.getVersion,
- new util.HashSet[String](mdl.getEnabledBuiltInTokens)
+ new util.HashSet[String](mdl.getEnabledBuiltInTokens),
+ new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava)
)
})
), cryptoKey)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index a72b6c0..028608a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -923,16 +923,18 @@
checkCollection("metadata", mdl.getMetadata)
checkCollection("restrictedCombinations", mdl.getRestrictedCombinations)
+ mdl.getElements.asScala.foreach(e => checkMandatoryString(e.getId,"element.id", MODEL_ELEMENT_ID_MAXLEN))
+
for ((elm, restrs: util.Set[String]) <- mdl.getRestrictedCombinations.asScala) {
if (elm != "nlpcraft:limit" && elm != "nlpcraft:sort" && elm != "nlpcraft:relation")
- throw new NCE(s"Unsupported restricting element ID [" +
+ throw new NCE(s"Unsupported restricting element [" +
s"mdlId=$mdlId, " +
- s"elemId=$elm" +
+ s"elmId=$elm" +
s"]. Only 'nlpcraft:limit', 'nlpcraft:sort', and 'nlpcraft:relation' are allowed.")
if (restrs.contains(elm))
throw new NCE(s"Element cannot be restricted to itself [" +
s"mdlId=$mdlId, " +
- s"elemId=$elm" +
+ s"elmId=$elm" +
s"]")
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index a2deee8..1f81711 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -250,14 +250,14 @@
if (res != null)
res.asScala.foreach(e => {
- val elemId = e.getElementId
+ val elmId = e.getElementId
val words = e.getWords
- if (elemId == null)
+ if (elmId == null)
throw new NCE(s"Custom model parser cannot return 'null' element ID.")
if (words == null || words.isEmpty)
- throw new NCE(s"Custom model parser cannot return empty custom tokens [elementId=$elemId]")
+ throw new NCE(s"Custom model parser cannot return empty custom tokens for element: $elmId")
val matchedToks = words.asScala.map(w =>
ns.find(t =>
@@ -266,10 +266,10 @@
)
// Checks element's tokens.
- if (!alreadyMarked(ns, elemId, matchedToks, matchedToks.map(_.index).sorted))
+ if (!alreadyMarked(ns, elmId, matchedToks, matchedToks.map(_.index).sorted))
mark(
ns,
- elem = mdl.elements.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")),
+ elem = mdl.elements.getOrElse(elmId, throw new NCE(s"Custom model parser returned unknown element: $elmId")),
toks = matchedToks,
direct = true,
metaOpt = Some(e.getMetadata.asScala.toMap)
@@ -577,16 +577,16 @@
// TODO: simplify, add tests, check model properties (sparse etc) for optimization.
/**
*
- * @param elemId
+ * @param elmId
* @param toks
* @param sliceToksIdxsSorted
*/
- private def alreadyMarked(ns: Sentence, elemId: String, toks: Seq[NlpToken], sliceToksIdxsSorted: Seq[Int]): Boolean = {
+ private def alreadyMarked(ns: Sentence, elmId: String, toks: Seq[NlpToken], sliceToksIdxsSorted: Seq[Int]): Boolean = {
lazy val toksIdxsSorted = toks.map(_.index).sorted
- sliceToksIdxsSorted.map(ns).forall(_.exists(n => n.noteType == elemId && n.sparsity == 0)) ||
+ sliceToksIdxsSorted.map(ns).forall(_.exists(n => n.noteType == elmId && n.sparsity == 0)) ||
toks.exists(_.exists(n =>
- n.noteType == elemId &&
+ n.noteType == elmId &&
(
(n.sparsity == 0 &&
(sliceToksIdxsSorted.containsSlice(n.tokenIndexes) || n.tokenIndexes.containsSlice(toksIdxsSorted))
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
index 16edd61..1b6001b 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
@@ -27,7 +27,8 @@
@NCMdoField id: String,
@NCMdoField name: String,
@NCMdoField version: String,
- @NCMdoField enabledBuiltInTokens: Set[String]
+ @NCMdoField enabledBuiltInTokens: Set[String],
+ @NCMdoField elementIds: Set[String]
) extends NCAnnotatedMdo[NCProbeModelMdo] {
override def hashCode(): Int = s"$id$name".hashCode()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
index 25340d7..e826dda 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
@@ -17,9 +17,7 @@
package org.apache.nlpcraft.server.opencensus
-import java.util.Collections
import io.opencensus.stats.Measure._
-import io.opencensus.stats.View.Name
import io.opencensus.stats._
import org.apache.nlpcraft.common.opencensus.NCOpenCensusStats
@@ -30,7 +28,8 @@
val M_HEALTH_MS: MeasureLong = MeasureLong.create("health_latency", "The latency of '/health' REST call", "ms")
val M_ASK_LATENCY_MS: MeasureLong = MeasureLong.create("ask_latency", "The latency of '/ask' REST call", "ms")
val M_CHECK_LATENCY_MS: MeasureLong = MeasureLong.create("check_latency", "The latency of '/check' REST call", "ms")
- val M_MODEL_SUGSYN_LATENCY_MS: MeasureLong = MeasureLong.create("model_inspect_latency", "The latency of '/model/inspect' REST call", "ms")
+ val M_MODEL_SUGSYN_LATENCY_MS: MeasureLong = MeasureLong.create("model_inspect_latency", "The latency of '/model/sugsyn' REST call", "ms")
+ val M_MODEL_SYNS_LATENCY_MS: MeasureLong = MeasureLong.create("model_synonyms_latency", "The latency of '/model/syns' REST call", "ms")
val M_CANCEL_LATENCY_MS: MeasureLong = MeasureLong.create("cancel_latency", "The latency of '/cancel' REST call", "ms")
val M_SIGNIN_LATENCY_MS: MeasureLong = MeasureLong.create("signin_latency", "The latency of '/signin' REST call", "ms")
val M_SIGNOUT_LATENCY_MS: MeasureLong = MeasureLong.create("signout_latency", "The latency of '/signout' REST call", "ms")
@@ -77,6 +76,7 @@
mkViews(M_COMPANY_TOKEN_LATENCY_MS, "company/token"),
mkViews(M_COMPANY_DELETE_LATENCY_MS, "company/delete"),
mkViews(M_MODEL_SUGSYN_LATENCY_MS, "model/sugsyn"),
+ mkViews(M_MODEL_SYNS_LATENCY_MS, "model/syns"),
mkViews(M_USER_ADD_LATENCY_MS, "user/add"),
mkViews(M_USER_GET_LATENCY_MS, "user/get"),
mkViews(M_USER_DELETE_LATENCY_MS, "user/delete"),
@@ -87,7 +87,6 @@
mkViews(M_FEEDBACK_ADD_LATENCY_MS, "feedback/add"),
mkViews(M_FEEDBACK_DELETE_LATENCY_MS, "feedback/delete"),
mkViews(M_FEEDBACK_GET_LATENCY_MS, "feedback/get"),
- mkViews(M_MODEL_SUGSYN_LATENCY_MS, "model/sugsyn"),
mkViews(M_PROBE_ALL_LATENCY_MS, "probe/all"),
mkViews(M_ROUND_TRIP_LATENCY_MS, "roundTrip/latdist"),
).flatten
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index f572b9f..feed48c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -23,6 +23,7 @@
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.crypto.NCCipher
+import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.pool.NCThreadPoolManager
@@ -45,7 +46,7 @@
import java.util.concurrent.ConcurrentHashMap
import scala.collection.mutable
import scala.concurrent.{ExecutionContext, Future, Promise}
-import scala.jdk.CollectionConverters.SetHasAsScala
+import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala}
import scala.util.{Failure, Success}
/**
@@ -53,7 +54,7 @@
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private final val TYPE_MODEL_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
+ private final val TYPE_JAVA_META = new TypeToken[JavaMeta]() {}.getType
// Type safe and eager configuration container.
private object Config extends NCConfigurable {
@@ -65,7 +66,7 @@
def reconnectTimeoutMs: Long = getLong(s"$pre.reconnectTimeoutMs")
def pingTimeoutMs: Long = getLong(s"$pre.pingTimeoutMs")
def soTimeoutMs: Int = getInt(s"$pre.soTimeoutMs")
-
+
/**
*
*/
@@ -156,6 +157,8 @@
// All probes pending complete handshake keyed by probe key.
@volatile private var pending: mutable.Map[ProbeKey, ProbeHolder] = _
+ @volatile private var modelsSynsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelElmsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
@volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
/**
@@ -178,6 +181,8 @@
"downlink" -> s"$dnHost:$dnPort"
)
+ modelsSynsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelElmsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
dnSrv = startServer("Downlink", dnHost, dnPort, downLinkHandler)
@@ -215,6 +220,8 @@
U.stopThread(dnSrv)
U.stopThread(upSrv)
+ modelsSynsInfo = null
+ modelElmsInfo = null
modelsInfo = null
ackStopped()
@@ -613,6 +620,7 @@
String,
String,
String,
+ java.util.Set[String],
java.util.Set[String]
)]]("PROBE_MODELS").
map {
@@ -620,18 +628,21 @@
mdlId,
mdlName,
mdlVer,
- enabledBuiltInToks
+ enabledBuiltInToks,
+ elmIds
) =>
require(mdlId != null)
require(mdlName != null)
require(mdlVer != null)
require(enabledBuiltInToks != null)
+ require(elmIds != null)
NCProbeModelMdo(
id = mdlId,
name = mdlName,
version = mdlVer,
- enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet
+ enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet,
+ elementIds = elmIds.asScala.toSet
)
}.toSet
@@ -682,6 +693,23 @@
}
}
}
+
+ /**
+ *
+ * @param probeMsg
+ * @param m
+ */
+ private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = {
+ val p = m.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
+ else
+ logger.warn(s"Message ignored: $probeMsg")
+ }
/**
* Processes the messages received from the probe.
@@ -707,13 +735,10 @@
typ match {
case "P2S_PING" => ()
- case "P2S_MODEL_INFO" =>
- val p = modelsInfo.remove(probeMsg.data[String]("reqGuid"))
+ case "P2S_MODEL_SYNS_INFO" => processJavaMetaMessage(probeMsg, modelsSynsInfo)
+ case "P2S_MODEL_ELEMENT_INFO" => processJavaMetaMessage(probeMsg, modelElmsInfo)
+ case "P2S_MODEL_INFO" => processJavaMetaMessage(probeMsg, modelsInfo)
- if (p != null)
- p.success(GSON.fromJson(probeMsg.data[String]("resp"), TYPE_MODEL_INFO_RESP))
- else
- logger.warn(s"Message ignored: $probeMsg")
case "P2S_ASK_RESULT" =>
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -763,6 +788,7 @@
NCErrorCodes.UNEXPECTED_ERROR
)
}
+
case _ =>
logger.error(s"Received unrecognized probe message (ignoring): $probeMsg")
}
@@ -966,6 +992,27 @@
}
/**
+ * Checks whether or not a data probe exists for given model element.
+ *
+ * @param compId Company ID for authentication purpose.
+ * @param mdlId Model ID.
+ * @param elmId Element ID.
+ * @param parent Optional parent span.
+ * @return
+ */
+ def existsForModelElement(compId: Long, mdlId: String, elmId: String, parent: Span = null): Boolean =
+ startScopedSpan(
+ "existsForModelElement", parent, "compId" -> compId, "mdlId" -> mdlId, "elmId" -> elmId
+ ) { _ =>
+ val authTok = getCompany(compId).authToken
+
+ probes.synchronized {
+ probes.filter(_._1.probeToken == authTok).values.
+ exists(_.probe.models.exists(p => p.id == mdlId && p.elementIds.contains(elmId)))
+ }
+ }
+
+ /**
*
* @param usrId User ID.
* @param mdlId Model ID.
@@ -1016,24 +1063,95 @@
/**
*
* @param mdlId
+ * @param msg
+ * @param holder
+ * @param parent
+ */
+ private def processModelDataRequest(
+ mdlId: String, msg: NCProbeMessage, holder: ConcurrentHashMap[String, Promise[JavaMeta]], parent: Span = null
+ ): Future[JavaMeta] = {
+ val p = Promise[JavaMeta]()
+
+ getProbeForModelId(mdlId) match {
+ case Some(probe) =>
+ holder.put(msg.getGuid, p)
+
+ sendToProbe(probe.probeKey, msg, parent)
+ case None =>
+ p.failure(new NCE(s"Probe not found for model: '$mdlId''"))
+ }
+
+ p.future
+ }
+
+ /**
+ *
+ * @param mdlId
* @param parent
* @return
*/
+ def getModelSynonymsInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelSynonymsInfo", parent, "mdlId" -> mdlId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_SYNS_INFO", "mdlId" -> mdlId),
+ modelsSynsInfo,
+ parent
+ )
+ }
+
+ /**
+ *
+ * @param mdlId
+ * @param elmId
+ * @param parent
+ * @return
+ */
+ def getModelElementInfo(mdlId: String, elmId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelElementInfo", parent, "mdlId" -> mdlId, "elmId" -> elmId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_ELEMENT_INFO", "mdlId" -> mdlId, "elmId" -> elmId),
+ modelElmsInfo,
+ parent
+ ).map(
+ res => {
+ require(
+ res.containsKey("synonyms") &&
+ res.containsKey("values") &&
+ res.containsKey("macros")
+ )
+
+ val macros = res.remove("macros").asInstanceOf[java.util.Map[String, String]].asScala
+ val syns = res.get("synonyms").asInstanceOf[java.util.List[String]].asScala
+ val vals = res.get("values").asInstanceOf[java.util.Map[String, java.util.List[String]]].asScala
+
+ val parser = NCMacroParser(macros.toList)
+
+ val synsExp = syns.flatMap(s => parser.expand(s)).sorted
+ val valsExp = vals.map(v => v._1 -> v._2.asScala.flatMap(s => parser.expand(s)).sorted.asJava).toMap
+
+ res.put("synonymsExp", synsExp.asJava)
+ res.put("valuesExp", valsExp.asJava)
+
+ // Add statistics.
+ res.put("synonymsExpCnt", Integer.valueOf(synsExp.size))
+ res.put("synonymsCnt", Integer.valueOf(syns.size))
+ res.put("synonymsExpRatePct", java.lang.Long.valueOf(Math.round(synsExp.size.toDouble * 100 / syns.size.toDouble)))
+
+ res
+ }
+ )
+ }
+
def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
- getProbeForModelId(mdlId) match {
- case Some(probe) =>
- val msg = NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId)
-
- val p = Promise[JavaMeta]()
-
- modelsInfo.put(msg.getGuid, p)
-
- sendToProbe(probe.probeKey, msg, parent)
-
- p.future
-
- case None => throw new NCE(s"Probe not found for model: '$mdlId''")
- }
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
+ modelsInfo,
+ parent
+ )
}
+
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 45ab892..8116d07 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -88,6 +88,7 @@
case class InvalidExternalUserId(usrExtId: String) extends InvalidArguments(s"External user ID is invalid or unknown: $usrExtId")
case class InvalidUserId(id: Long) extends InvalidArguments(s"User ID is invalid or unknown: $id")
case class InvalidModelId(id: String) extends InvalidArguments(s"Unknown model ID: $id")
+ case class InvalidModelOrElementId(mdlId: String, elmId: String) extends InvalidArguments(s"Unknown model element ID: $elmId")
case class AskReqHolder(
usrId: Long,
@@ -106,6 +107,7 @@
private final val STD_FIELD_LENGTHS = Map[String, Int](
"srvReqId" -> 32,
"mdlId" -> NCModelView.MODEL_ID_MAXLEN,
+ "elmId" -> NCModelView.MODEL_ELEMENT_ID_MAXLEN,
"country" -> 32,
"postalCode" -> 32,
@@ -469,7 +471,7 @@
*
* @return
*/
- protected def signin$(): Route = {
+ protected def $signin(): Route = {
case class Req$Signin$(
email: String,
passwd: String
@@ -504,7 +506,7 @@
*
* @return
*/
- protected def health$(): Route = {
+ protected def $health(): Route = {
case class Res$Health$(status: String)
implicit val resFmt: RootJsonFormat[Res$Health$] = jsonFormat1(Res$Health$)
@@ -518,7 +520,7 @@
*
* @return
*/
- protected def signout$(): Route = {
+ protected def $signout(): Route = {
case class Req$Signout$(
acsTok: String
)
@@ -632,7 +634,7 @@
*
* @return
*/
- protected def ask$(): Route =
+ protected def $ask(): Route =
ask0(
(h: AskReqHolder) => {
val newSrvReqId = NCQueryManager.asyncAsk(
@@ -663,7 +665,7 @@
*
* @return
*/
- protected def ask$Sync(): Route =
+ protected def $ask$Sync(): Route =
ask0(
(h: AskReqHolder) => {
val fut = NCQueryManager.futureAsk(
@@ -695,7 +697,7 @@
*
* @return
*/
- protected def cancel$(): Route = {
+ protected def $cancel(): Route = {
case class Req$Cancel$(
acsTok: String,
usrId: Option[Long],
@@ -734,7 +736,7 @@
*
* @return
*/
- protected def check$(): Route = {
+ protected def $check(): Route = {
case class Req$Check$(
acsTok: String,
usrId: Option[Long],
@@ -785,16 +787,16 @@
*
* @return
*/
- protected def sugsyn$(): Route = {
- case class Req$Model$Sugsyn$(
+ protected def $model$sugsyn(): Route = {
+ case class Req$Model$Sugsyn(
acsTok: String,
mdlId: String,
minScore: Option[Double]
)
- implicit val reqFmt: RootJsonFormat[Req$Model$Sugsyn$] = jsonFormat3(Req$Model$Sugsyn$)
+ implicit val reqFmt: RootJsonFormat[Req$Model$Sugsyn] = jsonFormat3(Req$Model$Sugsyn)
- entity(as[Req$Model$Sugsyn$]) { req =>
+ entity(as[Req$Model$Sugsyn]) { req =>
startScopedSpan(
"model$sugsyn",
"acsTok" -> req.acsTok,
@@ -819,11 +821,120 @@
}
}
+
/**
*
* @return
*/
- protected def clear$Conversation(): Route = {
+ protected def $model$syns(): Route = {
+ case class Req$Model$Syns(
+ acsTok: String,
+ mdlId: String,
+ elmId: String
+ )
+
+ implicit val reqFmt: RootJsonFormat[Req$Model$Syns] = jsonFormat3(Req$Model$Syns)
+
+ entity(as[Req$Model$Syns]) { req =>
+ startScopedSpan(
+ "model$syns",
+ "acsTok" -> req.acsTok,
+ "mdlId" -> req.mdlId,
+ "elmId" -> req.elmId) { span =>
+ checkLength("acsTok" -> req.acsTok, "mdlId" -> req.mdlId, "elmId" -> req.elmId)
+
+ val admUsr = authenticateAsAdmin(req.acsTok)
+ val compId = admUsr.companyId
+
+ if (!NCProbeManager.existsForModel(compId, req.mdlId))
+ throw InvalidModelId(req.mdlId)
+ if (!NCProbeManager.existsForModelElement(compId, req.mdlId, req.elmId))
+ throw InvalidModelOrElementId(req.mdlId, req.elmId)
+
+ val fut = NCProbeManager.getModelElementInfo(req.mdlId, req.elmId, span)
+
+ successWithJs(
+ fut.collect {
+ // We have to use Jackson (not spray) here to serialize 'result' field.
+ case res =>
+ require(
+ res.containsKey("synonyms") &&
+ res.containsKey("synonymsExp") &&
+ res.containsKey("synonymsExpRatePct") &&
+ res.containsKey("synonymsExpCnt") &&
+ res.containsKey("synonymsCnt") &&
+ res.containsKey("values") &&
+ res.containsKey("valuesExp")
+ )
+
+ toJs(
+ Map(
+ "status" -> API_OK.toString,
+ "synonymsCnt" -> res.get("synonymsCnt"),
+ "synonymsExpCnt" -> res.get("synonymsExpCnt"),
+ "synonymsExpRatePct" -> res.get("synonymsExpRatePct"),
+ "synonyms" -> res.get("synonyms"),
+ "synonymsExp" -> res.get("synonymsExp"),
+ "values" -> res.get("values"),
+ "valuesExp" -> res.get("valuesExp")
+ )
+ )
+ }
+ )
+ }
+ }
+ }
+
+
+ /**
+ *
+ * @return
+ */
+ protected def $model$info(): Route = {
+ case class Req$Model$Info(
+ acsTok: String,
+ mdlId: String
+ )
+
+ implicit val reqFmt: RootJsonFormat[Req$Model$Info] = jsonFormat2(Req$Model$Info)
+
+ entity(as[Req$Model$Info]) { req =>
+ startScopedSpan(
+ "model$syns",
+ "acsTok" -> req.acsTok,
+ "mdlId" -> req.mdlId
+ ) { span =>
+ checkLength("acsTok" -> req.acsTok, "mdlId" -> req.mdlId)
+
+ val admUsr = authenticateAsAdmin(req.acsTok)
+ val compId = admUsr.companyId
+
+ if (!NCProbeManager.existsForModel(compId, req.mdlId))
+ throw InvalidModelId(req.mdlId)
+
+ val fut = NCProbeManager.getModelInfo(req.mdlId, span)
+
+ successWithJs(
+ fut.collect {
+ // We have to use Jackson (not spray) here to serialize 'result' field.
+ case res =>
+ toJs(
+ Map(
+ "status" -> API_OK.toString,
+ "model" -> res
+ )
+ )
+ }
+ )
+ }
+ }
+ }
+
+ /**
+ *
+ * @return
+ */
+ protected def $clear$Conversation(): Route = {
case class Req$Clear$Conversation(
acsTok: String,
mdlId: String,
@@ -868,7 +979,7 @@
*
* @return
*/
- protected def clear$Dialog(): Route = {
+ protected def $clear$Dialog(): Route = {
case class Req$Clear$Dialog(
acsTok: String,
mdlId: String,
@@ -913,7 +1024,7 @@
*
* @return
*/
- protected def company$Add(): Route = {
+ protected def $company$Add(): Route = {
case class Req$Company$Add(
acsTok: String,
// New company.
@@ -993,7 +1104,7 @@
*
* @return
*/
- protected def company$Get(): Route = {
+ protected def $company$Get(): Route = {
case class Req$Company$Get(
acsTok: String
)
@@ -1043,7 +1154,7 @@
*
* @return
*/
- protected def company$Update(): Route = {
+ protected def $company$Update(): Route = {
case class Req$Company$Update(
// Caller.
acsTok: String,
@@ -1104,7 +1215,7 @@
*
* @return
*/
- protected def feedback$Add(): Route = {
+ protected def $feedback$Add(): Route = {
case class Req$Feedback$Add(
acsTok: String,
usrId : Option[Long],
@@ -1157,7 +1268,7 @@
*
* @return
*/
- protected def feedback$Delete(): Route = {
+ protected def $feedback$Delete(): Route = {
case class Req$Feedback$Delete(
acsTok: String,
// Feedback IDs to delete (optional).
@@ -1214,7 +1325,7 @@
*
* @return
*/
- protected def feedback$All(): Route = {
+ protected def $feedback$All(): Route = {
case class Req$Feedback$All(
acsTok: String,
usrId: Option[Long],
@@ -1284,7 +1395,7 @@
*
* @return
*/
- protected def company$Token$Reset(): Route = {
+ protected def $company$Token$Reset(): Route = {
case class Req$Company$Token$Reset(
// Caller.
acsTok: String
@@ -1316,7 +1427,7 @@
*
* @return
*/
- protected def company$Delete(): Route = {
+ protected def $company$Delete(): Route = {
case class Req$Company$Delete(
// Caller.
acsTok: String
@@ -1347,7 +1458,7 @@
*
* @return
*/
- protected def user$Add(): Route = {
+ protected def $user$Add(): Route = {
case class Req$User$Add(
// Caller.
acsTok: String,
@@ -1409,7 +1520,7 @@
*
* @return
*/
- protected def user$Update(): Route = {
+ protected def $user$Update(): Route = {
case class Req$User$Update(
// Caller.
acsTok: String,
@@ -1460,7 +1571,7 @@
*
* @return
*/
- protected def user$Delete(): Route = {
+ protected def $user$Delete(): Route = {
case class Req$User$Delete(
acsTok: String,
id: Option[Long],
@@ -1525,7 +1636,7 @@
*
* @return
*/
- protected def user$Admin(): Route = {
+ protected def $user$Admin(): Route = {
case class Req$User$Admin(
acsTok: String,
id: Option[Long],
@@ -1568,7 +1679,7 @@
*
* @return
*/
- protected def user$Password$Reset(): Route = {
+ protected def $user$Passwd$Reset(): Route = {
case class Req$User$Password$Reset(
// Caller.
acsTok: String,
@@ -1603,7 +1714,7 @@
*
* @return
*/
- protected def user$All(): Route = {
+ protected def $user$All(): Route = {
case class Req$User$All(
// Caller.
acsTok: String
@@ -1658,7 +1769,7 @@
*
* @return
*/
- protected def user$Get(): Route = {
+ protected def $user$Get(): Route = {
case class Req$User$Get(
// Caller.
acsTok: String,
@@ -1713,7 +1824,7 @@
*
* @return
*/
- protected def probe$All(): Route = {
+ protected def $probe$All(): Route = {
case class Req$Probe$All(
acsTok: String
)
@@ -1721,6 +1832,7 @@
id: String,
name: String,
version: String,
+ elementIds: Set[String],
enabledBuiltInTokens: Set[String]
)
case class Probe_Probe$All(
@@ -1750,7 +1862,7 @@
)
implicit val reqFmt: RootJsonFormat[Req$Probe$All] = jsonFormat1(Req$Probe$All)
- implicit val mdlFmt: RootJsonFormat[Model_Probe$All] = jsonFormat4(Model_Probe$All)
+ implicit val mdlFmt: RootJsonFormat[Model_Probe$All] = jsonFormat5(Model_Probe$All)
implicit val probFmt: RootJsonFormat[Probe_Probe$All] = jsonFormat19(Probe_Probe$All)
implicit val resFmt: RootJsonFormat[Res$Probe$All] = jsonFormat2(Res$Probe$All)
@@ -1783,6 +1895,7 @@
m.id,
m.name,
m.version,
+ m.elementIds,
m.enabledBuiltInTokens
))
))
@@ -1947,38 +2060,40 @@
corsHandler (
get {
withRequestTimeoutResponse(_ => timeoutResp) {
- path(API / "health") { health$() } // Also duplicated for POST.
+ path(API / "health") { $health() } // Also duplicated for POST.
}
} ~
post {
encodeResponseWith(Coders.NoCoding, Coders.Gzip) {
withRequestTimeoutResponse(_ => timeoutResp) {
- path(API / "health") { health$() } // Duplicate for POST.
- path(API / "signin") { withMetric(M_SIGNIN_LATENCY_MS, signin$) } ~
- path(API / "signout") { withMetric(M_SIGNOUT_LATENCY_MS, signout$) } ~
- path(API / "cancel") { withMetric(M_CANCEL_LATENCY_MS, cancel$) } ~
- path(API / "check") { withMetric(M_CHECK_LATENCY_MS, check$) } ~
- path(API / "clear"/ "conversation") { withMetric(M_CLEAR_CONV_LATENCY_MS, clear$Conversation) } ~
- path(API / "clear"/ "dialog") { withMetric(M_CLEAR_DIALOG_LATENCY_MS, clear$Dialog) } ~
- path(API / "company"/ "add") { withMetric(M_COMPANY_ADD_LATENCY_MS, company$Add) } ~
- path(API / "company"/ "get") { withMetric(M_COMPANY_GET_LATENCY_MS, company$Get) } ~
- path(API / "company" / "update") { withMetric(M_COMPANY_UPDATE_LATENCY_MS, company$Update) } ~
- path(API / "company" / "token" / "reset") { withMetric(M_COMPANY_TOKEN_LATENCY_MS, company$Token$Reset) } ~
- path(API / "company" / "delete") { withMetric(M_COMPANY_DELETE_LATENCY_MS, company$Delete) } ~
- path(API / "user" / "get") { withMetric(M_USER_GET_LATENCY_MS, user$Get) } ~
- path(API / "user" / "add") { withMetric(M_USER_ADD_LATENCY_MS, user$Add) } ~
- path(API / "user" / "update") { withMetric(M_USER_UPDATE_LATENCY_MS, user$Update) } ~
- path(API / "user" / "delete") { withMetric(M_USER_DELETE_LATENCY_MS, user$Delete) } ~
- path(API / "user" / "admin") { withMetric(M_USER_ADMIN_LATENCY_MS, user$Admin) } ~
- path(API / "user" / "passwd" / "reset") { withMetric(M_USER_PASSWD_RESET_LATENCY_MS, user$Password$Reset) } ~
- path(API / "user" / "all") { withMetric(M_USER_ALL_LATENCY_MS, user$All) } ~
- path(API / "feedback"/ "add") { withMetric(M_FEEDBACK_ADD_LATENCY_MS, feedback$Add) } ~
- path(API / "feedback"/ "all") { withMetric(M_FEEDBACK_GET_LATENCY_MS, feedback$All) } ~
- path(API / "feedback" / "delete") { withMetric(M_FEEDBACK_DELETE_LATENCY_MS, feedback$Delete) } ~
- path(API / "probe" / "all") { withMetric(M_PROBE_ALL_LATENCY_MS, probe$All) } ~
- path(API / "model" / "sugsyn") { withMetric(M_MODEL_SUGSYN_LATENCY_MS, sugsyn$) } ~
- path(API / "ask") { withMetric(M_ASK_LATENCY_MS, ask$) } ~
- path(API / "ask" / "sync") { withMetric(M_ASK_SYNC_LATENCY_MS, ask$Sync) }
+ path(API / "health") { $health() } // Duplicate for POST.
+ path(API / "signin") { withMetric(M_SIGNIN_LATENCY_MS, $signin) } ~
+ path(API / "signout") { withMetric(M_SIGNOUT_LATENCY_MS, $signout) } ~
+ path(API / "cancel") { withMetric(M_CANCEL_LATENCY_MS, $cancel) } ~
+ path(API / "check") { withMetric(M_CHECK_LATENCY_MS, $check) } ~
+ path(API / "clear"/ "conversation") { withMetric(M_CLEAR_CONV_LATENCY_MS, $clear$Conversation) } ~
+ path(API / "clear"/ "dialog") { withMetric(M_CLEAR_DIALOG_LATENCY_MS, $clear$Dialog) } ~
+ path(API / "company"/ "add") { withMetric(M_COMPANY_ADD_LATENCY_MS, $company$Add) } ~
+ path(API / "company"/ "get") { withMetric(M_COMPANY_GET_LATENCY_MS, $company$Get) } ~
+ path(API / "company" / "update") { withMetric(M_COMPANY_UPDATE_LATENCY_MS, $company$Update) } ~
+ path(API / "company" / "token" / "reset") { withMetric(M_COMPANY_TOKEN_LATENCY_MS, $company$Token$Reset) } ~
+ path(API / "company" / "delete") { withMetric(M_COMPANY_DELETE_LATENCY_MS, $company$Delete) } ~
+ path(API / "user" / "get") { withMetric(M_USER_GET_LATENCY_MS, $user$Get) } ~
+ path(API / "user" / "add") { withMetric(M_USER_ADD_LATENCY_MS, $user$Add) } ~
+ path(API / "user" / "update") { withMetric(M_USER_UPDATE_LATENCY_MS, $user$Update) } ~
+ path(API / "user" / "delete") { withMetric(M_USER_DELETE_LATENCY_MS, $user$Delete) } ~
+ path(API / "user" / "admin") { withMetric(M_USER_ADMIN_LATENCY_MS, $user$Admin) } ~
+ path(API / "user" / "passwd" / "reset") { withMetric(M_USER_PASSWD_RESET_LATENCY_MS, $user$Passwd$Reset) } ~
+ path(API / "user" / "all") { withMetric(M_USER_ALL_LATENCY_MS, $user$All) } ~
+ path(API / "feedback"/ "add") { withMetric(M_FEEDBACK_ADD_LATENCY_MS, $feedback$Add) } ~
+ path(API / "feedback"/ "all") { withMetric(M_FEEDBACK_GET_LATENCY_MS, $feedback$All) } ~
+ path(API / "feedback" / "delete") { withMetric(M_FEEDBACK_DELETE_LATENCY_MS, $feedback$Delete) } ~
+ path(API / "probe" / "all") { withMetric(M_PROBE_ALL_LATENCY_MS, $probe$All) } ~
+ path(API / "model" / "sugsyn") { withMetric(M_MODEL_SUGSYN_LATENCY_MS, $model$sugsyn) } ~
+ path(API / "model" / "syns") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$syns) } ~
+ path(API / "model" / "info") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$info) } ~
+ path(API / "ask") { withMetric(M_ASK_LATENCY_MS, $ask) } ~
+ path(API / "ask" / "sync") { withMetric(M_ASK_SYNC_LATENCY_MS, $ask$Sync) }
}
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 76af8a0..d89ba98 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -143,7 +143,7 @@
val promise = Promise[NCSuggestSynonymResult]()
- NCProbeManager.getModelInfo(mdlId, parent).onComplete {
+ NCProbeManager.getModelSynonymsInfo(mdlId, parent).onComplete {
case Success(m) =>
try {
require(
@@ -218,13 +218,13 @@
}).
toMap
- val elemSyns =
- mdlSyns.map { case (elemId, syns) => elemId -> syns.flatMap(parser.expand) }.
+ val elmSyns =
+ mdlSyns.map { case (elmId, syns) => elmId -> syns.flatMap(parser.expand) }.
map { case (id, seq) => id -> seq.map(txt => split(txt).map(p => Word(p, toStemWord(p)))) }
val allReqs =
- elemSyns.map {
- case (elemId, syns) =>
+ elmSyns.map {
+ case (elmId, syns) =>
// Current implementation supports suggestions only for single words synonyms.
val normSyns: Seq[Seq[Word]] = syns.filter(_.size == 1)
val synsStems = normSyns.map(_.map(_.stem))
@@ -248,7 +248,7 @@
}
}.mkString(" "),
ex = exWords.mkString(" "),
- elmId = elemId,
+ elmId = elmId,
index = idx
)
}
@@ -257,20 +257,20 @@
yield mkRequestData(idx, synStems, i)).distinct
}
- elemId -> reqs.toSet
+ elmId -> reqs.toSet
}.filter(_._2.nonEmpty)
- val noExElems =
+ val noExElms =
mdlSyns.
- filter { case (elemId, syns) => syns.nonEmpty && !allReqs.contains(elemId) }.
- map { case (elemId, _) => elemId }
+ filter { case (elmId, syns) => syns.nonEmpty && !allReqs.contains(elmId) }.
+ map { case (elmId, _) => elmId }
- if (noExElems.nonEmpty)
+ if (noExElms.nonEmpty)
warns += s"Elements do not have *single word* synonyms in their @NCIntentSample or @NCIntentSampleRef annotations - " +
- s"no suggestion can be made: ${noExElems.mkString(", ")}"
+ s"no suggestion can be made: ${noExElms.mkString(", ")}"
val allReqsCnt = allReqs.map(_._2.size).sum
- val allSynsCnt = elemSyns.map(_._2.size).sum
+ val allSynsCnt = elmSyns.map(_._2.size).sum
logger.trace(s"Request is going to execute on 'ctxword' server [" +
s"exs=${exs.size}, " +
@@ -289,7 +289,7 @@
val cli = HttpClients.createDefault
val err = new AtomicReference[Throwable]()
- for ((elemId, reqs) <- allReqs; batch <- reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
+ for ((elmId, reqs) <- allReqs; batch <- reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
U.asFuture(
_ => {
val post = new HttpPost(url)
@@ -322,7 +322,7 @@
logger.debug(s"Executed: $i requests...")
allSgsts.
- computeIfAbsent(elemId, (_: String) => new CopyOnWriteArrayList[Suggestion]()).
+ computeIfAbsent(elmId, (_: String) => new CopyOnWriteArrayList[Suggestion]()).
addAll(resps.flatten.asJava)
if (i == allReqsCnt)
@@ -342,13 +342,13 @@
if (err.get() != null)
throw new NCE("Error while working with 'ctxword' server.", err.get())
- val allSynsStems = elemSyns.flatMap(_._2).toSeq.flatten.map(_.stem).toSet
+ val allSynsStems = elmSyns.flatMap(_._2).toSeq.flatten.map(_.stem).toSet
val nonEmptySgsts = allSgsts.asScala.map(p => p._1 -> p._2.asScala).filter(_._2.nonEmpty)
val res = mutable.HashMap.empty[String, mutable.ArrayBuffer[SuggestionResult]]
- nonEmptySgsts.foreach { case (elemId, elemSgsts) =>
+ nonEmptySgsts.foreach { case (elmId, elemSgsts) =>
elemSgsts.
map(sgst => (sgst, toStem(sgst.word))).
groupBy { case (_, stem) => stem }.
@@ -366,12 +366,12 @@
zipWithIndex.
foreach { case ((word, _, sumFactor), _) =>
val seq =
- res.get(elemId) match {
+ res.get(elmId) match {
case Some(seq) => seq
case None =>
val buf = mutable.ArrayBuffer.empty[SuggestionResult]
- res += elemId -> buf
+ res += elmId -> buf
buf
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestElement.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestElement.scala
index dc465d8..3598d2e 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestElement.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestElement.scala
@@ -17,7 +17,7 @@
package org.apache.nlpcraft
-import org.apache.nlpcraft.model.NCElement
+import org.apache.nlpcraft.model.{NCElement, NCValue}
import java.util
import scala.jdk.CollectionConverters.{SeqHasAsJava, SetHasAsJava}
@@ -27,8 +27,11 @@
* Simple test element.
*/
case class NCTestElement(id: String, syns: String*) extends NCElement {
+ private val values = new util.ArrayList[NCValue]
+
override def getId: String = id
override def getSynonyms: util.List[String] = (syns :+ id).asJava
+ override def getValues: util.List[NCValue] = values
}
/**
@@ -38,4 +41,19 @@
private def to(e: NCTestElement): NCElement = e
implicit def to(set: Set[NCTestElement]): util.Set[NCElement] = set.map(to).asJava
+
+ def apply(id: String, syns: Seq[String], vals: Map[String, Seq[String]]): NCTestElement = {
+ val e = NCTestElement(id, syns :_*)
+
+ e.getValues.addAll(
+ vals.map { case (value, syns) =>
+ new NCValue {
+ override def getName: String = value
+ override def getSynonyms: util.List[String] = syns.asJava
+ }
+ }.toSeq.asJava
+ )
+
+ e
+ }
}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index d47a297..d7d22b8 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -17,19 +17,21 @@
package org.apache.nlpcraft.server.rest
-import org.apache.nlpcraft.NCTestEnvironment
+import org.apache.nlpcraft.model.NCElement
+import org.apache.nlpcraft.{NCTestElement, NCTestEnvironment}
import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.Test
-import scala.jdk.CollectionConverters.ListHasAsScala
+import java.util
+import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, SetHasAsJava, SetHasAsScala}
/**
* Note that context word server should be started.
*/
@NCTestEnvironment(model = classOf[RestTestModel], startClient = false)
-class NCRestModelSpec extends NCRestSpec {
+class NCRestModelSpec1 extends NCRestSpec {
@Test
- def test(): Unit = {
+ def testSugsyn(): Unit = {
def extract(data: JList[java.util.Map[String, Object]]): Seq[Double] =
data.asScala.map(_.get("score").asInstanceOf[Number].doubleValue()).toSeq
@@ -54,5 +56,70 @@
assertTrue(scores.forall(s => s >= 0.5 && s <= 1))
})
)
+
+ postError("model/sugsyn", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN")
+ postError("model/sugsyn", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "minScore" -> 2)
+ postError("model/sugsyn", 400, "NC_ERROR")
+ }
+}
+
+class RestTestModelExt extends RestTestModel {
+ override def getMacros: util.Map[String, String] = {
+ Map(
+ "<M1>" -> "mtest1 {x|_}",
+ "<M2>" -> "<M1> mtest2 {mtest3|_}"
+ ).asJava
+ }
+
+ override def getElements: util.Set[NCElement] = {
+ (
+ super.getElements.asScala ++
+ Set(
+ NCTestElement("eExt1", "<M1>", "<M1> more"),
+ NCTestElement("eExt2", Seq("<M1>", "<M1> more"), Map("v1"-> Seq("<M2>", "<M2> more"), "v2" -> Seq("<M2>")))
+ )
+ ).asJava
+ }
+}
+
+/**
+ *
+ */
+@NCTestEnvironment(model = classOf[RestTestModelExt], startClient = false)
+class NCRestModelSpec2 extends NCRestSpec {
+ @Test
+ def testSyns(): Unit = {
+ // Note that checked values are valid for current configuration of `RestTestModelExt` model.
+ def post0(elemId: String, valsShouldBe: Boolean): Unit =
+ post("model/syns", "mdlId" -> "rest.test.model", "elmId" -> elemId)(
+ ("$.status", (status: String) => assertEquals("API_OK", status)),
+ ("$.synonyms", (data: ResponseList) => assertTrue(!data.isEmpty)),
+ ("$.synonymsExp", (data: ResponseList) => assertTrue(!data.isEmpty)),
+ ("$.values", (data: java.util.Map[Object, Object]) =>
+ if (valsShouldBe) assertTrue(!data.isEmpty) else assertTrue(data.isEmpty)),
+ ("$.valuesExp", (data: java.util.Map[Object, Object]) =>
+ if (valsShouldBe) assertTrue(!data.isEmpty) else assertTrue(data.isEmpty)
+ )
+ )
+
+ post0("eExt1", valsShouldBe = false)
+ post0("eExt2", valsShouldBe = true)
+
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN", "elmId" -> "UNKNOWN")
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> "UNKNOWN")
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> ("A" * 33), "elmId" -> "UNKNOWN")
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> ("A" * 65))
+ postError("model/syns", 400, "NC_ERROR", "mdlId" -> "rest.test.model")
+ }
+
+ @Test
+ def testModelInfo(): Unit = {
+ post("model/info", "mdlId" -> "rest.test.model")(
+ ("$.status", (status: String) => assertEquals("API_OK", status)),
+ ("$.model", (data: java.util.Map[Object, Object]) => assertTrue(!data.isEmpty))
+ )
+
+ postError("model/info", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN")
+ postError("model/info", 400, "NC_ERROR")
}
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
index 03bf811..cb7d075 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
@@ -32,7 +32,6 @@
import org.junit.jupiter.api.{AfterEach, BeforeEach}
import java.util
-import java.util.{List => JList}
import java.util.UUID
import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/RestTestModel.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/RestTestModel.scala
index 8099e39..a46ee0f 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/RestTestModel.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/RestTestModel.scala
@@ -18,7 +18,7 @@
package org.apache.nlpcraft.server.rest
import org.apache.nlpcraft.NCTestElement
-import org.apache.nlpcraft.model.{NCElement, NCIntent, NCIntentSample, NCModelAdapter, NCResult}
+import org.apache.nlpcraft.model.{NCElement, NCIntent, NCIntentSample, NCModelAdapter, NCResult, NCValue}
import java.util
@@ -49,7 +49,8 @@
NCTestElement("a"),
NCTestElement("b"),
NCTestElement("x", "cat"),
- NCTestElement("meta")
+ NCTestElement("meta"),
+ NCTestElement("valElem", Seq("valElem1"), Map("v1"-> Seq("v11", "v12"), "v2" -> Seq("v21")))
)
@NCIntent("intent=onA term(t)={tok_id() == 'a'}")
diff --git a/openapi/nlpcraft_swagger.yml b/openapi/nlpcraft_swagger.yml
index 4b0be1e..4391cb3 100644
--- a/openapi/nlpcraft_swagger.yml
+++ b/openapi/nlpcraft_swagger.yml
@@ -234,14 +234,138 @@
description: Failed operation.
schema:
$ref: '#/definitions/Error'
+ /model/syns:
+ post:
+ tags:
+ - Tools
+ summary: Gets model element expanded synonyms and values.
+ description: >-
+ Gets model element expanded synonyms and values.
+ Administrative privileges required.
+ operationId: syns
+ parameters:
+ - in: body
+ name: Payload body
+ description: JSON request.
+ required: true
+ schema:
+ type: object
+ required:
+ - acsTok
+ - mdlId
+ - elmId
+ properties:
+ acsTok:
+ type: string
+ description: Access token obtain via '/signin' call.
+ maxLength: 256
+ mdlId:
+ type: string
+ maxLength: 32
+ description: ID of the model to run get elements synonyms and values.
+ elmId:
+ type: string
+ maxLength: 64
+ description: ID of the model element to get its synonyms and values.
+ responses:
+ '200':
+ description: Successful operation.
+ schema:
+ type: object
+ required:
+ - status
+ - synonyms
+ - values
+ - synonymsExp
+ - valuesExp
+ properties:
+ status:
+ type: string
+ description: Status code of this operation.
+ enum:
+ - API_OK
+ synonyms:
+ type: object
+ description: List of individual synomyms as they are specified in the model.
+ synonymsExp:
+ type: object
+ description: List of expanded synomyms as the model uses them (before stemmatization).
+ values:
+ type: object
+ description: >-
+ Map of element values, if any, as they are specified in the model.
+ Map key is the value name.
+ Map value is a list of synonyms for that value.
+ Empty if element doesn't have values.
+ valuesExp:
+ type: object
+ description: >-
+ Map of element values, if any, as the model uses them (before stemmatization).
+ Map key is the value name.
+ Map value is a list of expanded
+ synonyms for that value.
+ Empty if element doesn't have values.
+ '400':
+ description: Failed operation.
+ schema:
+ $ref: '#/definitions/Error'
+ /model/info:
+ post:
+ tags:
+ - Tools
+ summary: Gets model information.
+ description: >-
+ Gets model configuration properties and elements.
+ Administrative privileges required.
+ operationId: syns
+ parameters:
+ - in: body
+ name: Payload body
+ description: JSON request.
+ required: true
+ schema:
+ type: object
+ required:
+ - acsTok
+ - mdlId
+ properties:
+ acsTok:
+ type: string
+ description: Access token obtain via '/signin' call.
+ maxLength: 256
+ mdlId:
+ type: string
+ maxLength: 32
+ description: ID of the model to get information for.
+ responses:
+ '200':
+ description: Successful operation.
+ schema:
+ type: object
+ required:
+ - status
+ - model
+ properties:
+ status:
+ type: string
+ description: Status code of this operation.
+ enum:
+ - API_OK
+ model:
+ type: object
+ description: Object representing model configuration properties and elements.
+ '400':
+ description: Failed operation.
+ schema:
+ $ref: '#/definitions/Error'
/check:
post:
tags:
- Asking
summary: Gets status and result of submitted requests.
description: >-
- Gets the status and result (OK or failure) of the previously submitted requests.
- Request statuses returned sorted by their registration time, starting from newest.
+ Gets the status and result (OK or failure) of the previously submitted requests.
+ Request statuses returned sorted by their registration time, starting from newest.
operationId: check
parameters:
- in: body
