Merge branch 'master' into NLPCRAFT-376
diff --git a/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala b/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala
index 181ae8b..2163a75 100644
--- a/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala
+++ b/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala
@@ -400,7 +400,6 @@
* cleared between user questions, except for the obvious clarifying questions. We assume that question is being
* clarified if its tokens satisfy one of criteria:
* - all these tokens are values (What about 'Exotic Liquids')
- * - all these tokens are columns (Give me 'last name')
* - new token is single date token (What about 'tomorrow')
* <p>
* If new sentence tokens satisfied any of these criteria,
@@ -418,10 +417,9 @@
case Some(col) => col.getValue != null
case None => false
}
- def isColumn(t: NCToken): Boolean = findAnyColumnTokenOpt(t).isDefined
def isDate(t: NCToken): Boolean = t.getId == "nlpcraft:date"
- val ok = toks.forall(isValue) || toks.forall(isColumn) || toks.size == 1 && isDate(toks.head)
+ val ok = toks.forall(isValue) || toks.size == 1 && isDate(toks.head)
if (!ok) {
m.getContext.getConversation.clearStm(_ => true)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
index e16e8a5..ba9daf5 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/solver/NCIntentSolver.scala
@@ -28,9 +28,9 @@
import org.apache.nlpcraft.model.{NCContext, NCIntentMatch, NCIntentSkip, NCModel, NCRejection, NCResult, NCToken, NCVariant}
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
-import java.util.{List => JList}
-
-import scala.jdk.CollectionConverters.SeqHasAsJava
+import java.util.{Collections, List => JList}
+import scala.collection.mutable
+import scala.jdk.CollectionConverters.{ListHasAsScala, SeqHasAsJava}
/**
* Front-end for intent solver.
@@ -95,10 +95,23 @@
for (res <- results if res != null) {
try {
i += 1
-
+
+ val allConvToks = ctx.getConversation.getTokens.asScala
+ val nonConvToks = res.groups.flatMap(_.tokens).filterNot(allConvToks.contains)
+
+ val intentToks =
+ res.groups.map(_.tokens).map(toks => {
+ toks.filter(allConvToks.contains).foreach(convTok =>
+ fixBuiltTokensMeta(convTok, nonConvToks, allConvToks))
+
+ toks.asJava
+ }).asJava
+
+ ctx.getConversation.getTokens
+
val intentMatch: NCIntentMatch = new NCMetadataAdapter with NCIntentMatch {
override val getContext: NCContext = ctx
- override val getIntentTokens: JList[JList[NCToken]] = res.groups.map(_.tokens.asJava).asJava
+ override val getIntentTokens: JList[JList[NCToken]] = intentToks
override val getVariant: NCVariant = new NCVariantImpl(res.variant.tokens)
override val getIntentId: String = res.intentId
override def getTermTokens(idx: Int): JList[NCToken] = res.groups(idx).tokens.asJava
@@ -161,4 +174,146 @@
throw new NCRejection("No matching intent found - all intents were skipped.")
}
+
+ /**
+ *
+ * @param convTok
+ * @param nonConvToks
+ * @param allConvToks
+ */
+ @throws[NCE]
+ private def fixBuiltTokensMeta(convTok: NCToken, nonConvToks: Seq[NCToken], allConvToks: Seq[NCToken]): Unit = {
+ def isReference(tok: NCToken, id: String, idx: Int): Boolean = tok.getId == id && tok.getIndex == idx
+
+ /**
+ * Gets new references candidates.
+ *
+ * 1. It finds references in conversation. It should be here because not found among non conversation tokens.
+ * 2. Next, it finds common group for all found conversation's references, it also should be.
+ * 3. Next, for found group, it tries to find actual tokens with this group among non-conversation tokens.
+ * If these non-conversation tokens found, they should be validated and returned,
+ * If not found - conversation tokens returned.
+ *
+ * @param refId Reference token ID.
+ * @param refIdxs Reference indexes.
+ * @param validate Validate predicate.
+ */
+ @throws[NCE]
+ def getForRecalc(refId: String, refIdxs: Seq[Int], validate: Seq[NCToken] => Boolean): Seq[NCToken] = {
+ val convRefs = allConvToks.filter(_.getId == refId)
+
+ if (convRefs.map(_.getIndex).sorted != refIdxs.sorted)
+ throw new NCE(s"Conversation references are not found [id=$refId, indexes=${refIdxs.mkString(", ")}]")
+
+ val convGs = convRefs.map(_.getGroups.asScala)
+ val commonConvGs = convGs.foldLeft(convGs.head)((g1, g2) => g1.intersect(g2))
+
+ if (commonConvGs.isEmpty)
+ throw new NCE(s"Conversation references don't have common group [id=$refId]")
+
+ val actualNonConvRefs = nonConvToks.filter(_.getGroups.asScala.exists(commonConvGs.contains))
+
+ if (actualNonConvRefs.nonEmpty) {
+ if (!validate(actualNonConvRefs))
+ throw new NCE(
+ s"Actual valid variant references are not found for recalculation [" +
+ s"id=$refId, " +
+ s"actualNonConvRefs=${actualNonConvRefs.map(p => s"${p.getOriginalText}(${p.getId}-${p.getIndex})").mkString(",")}" +
+ s"]"
+ )
+
+ actualNonConvRefs
+ }
+ else
+ convRefs
+ }
+
+ convTok.getId match {
+ case "nlpcraft:sort" =>
+ def getNotNullSeq[T](tok: NCToken, name: String): Seq[T] = {
+ val list = tok.meta[JList[T]](name)
+
+ if (list == null) Seq.empty else list.asScala
+ }
+
+ def process(notesName: String, idxsName: String): Unit = {
+ val refIds: Seq[String] = getNotNullSeq(convTok, s"nlpcraft:sort:$notesName")
+ val refIdxs: Seq[Int] = getNotNullSeq(convTok, s"nlpcraft:sort:$idxsName")
+
+ require(refIds.length == refIdxs.length)
+
+ // Can be empty section.
+ if (refIds.nonEmpty) {
+ var data = mutable.ArrayBuffer.empty[(String, Int)]
+ val notFound = mutable.ArrayBuffer.empty[(String, Int)]
+
+ // Sort references can be different types.
+ // Part of them can be in conversation, part of them - in actual variant.
+ refIds.zip(refIdxs).map { case (refId, refIdx) =>
+ val seq =
+ nonConvToks.find(isReference(_, refId, refIdx)) match {
+ case Some(_) => data
+ case None => notFound
+ }
+
+ seq += refId -> refIdx
+ }
+
+ if (notFound.nonEmpty) {
+ notFound.
+ groupBy { case (refId, _) => refId }.
+ map { case (refId, data) => refId -> data.map(_._2) }.
+ foreach { case (refId, refIdxs) =>
+ getForRecalc(refId, refIdxs, _.size == refIdxs.size).
+ foreach(t => data += t.getId -> t.getIndex)
+ }
+
+ data = data.sortBy(_._2)
+
+ convTok.getMetadata.put(s"nlpcraft:sort:$notesName", data.map(_._1).asJava)
+ convTok.getMetadata.put(s"nlpcraft:sort:$idxsName", data.map(_._2).asJava)
+ }
+ }
+ }
+
+ process("bynotes", "byindexes")
+ process("subjnotes", "subjindexes")
+ case "nlpcraft:limit" =>
+ val refId = convTok.meta[String]("nlpcraft:limit:note")
+ val refIdxs = convTok.meta[JList[Int]]("nlpcraft:limit:indexes").asScala
+
+ require(refIdxs.size == 1)
+
+ val refIdx = refIdxs.head
+
+ if (!nonConvToks.exists(isReference(_, refId, refIdx))) {
+ val ref = getForRecalc(refId, Seq(refIdx), _.size == 1).head
+
+ convTok.getMetadata.put(s"nlpcraft:limit:note", ref.getId)
+ convTok.getMetadata.put(s"nlpcraft:limit:indexes", Collections.singletonList(ref.getIndex))
+ }
+
+ case "nlpcraft:relation" =>
+ val refId = convTok.meta[String]("nlpcraft:relation:note")
+ val refIdxs = convTok.meta[JList[Int]]("nlpcraft:relation:indexes").asScala.sorted
+
+ val nonConvRefs = nonConvToks.filter(t => t.getId == refId && refIdxs.contains(t.getIndex))
+
+ if (nonConvRefs.nonEmpty && nonConvRefs.size != refIdxs.size)
+ throw new NCE(s"References are not found [id=$refId, indexes=${refIdxs.mkString(", ")}]")
+
+ if (nonConvRefs.isEmpty) {
+ val refs = getForRecalc(refId, refIdxs, _.size == refIdxs.size)
+ val refsIds = refs.map(_.getId).distinct
+
+ if (refsIds.size != 1)
+ throw new NCE(s"Valid variant references are not found [id=$refId]")
+
+ convTok.getMetadata.put(s"nlpcraft:relation:note", refsIds.head)
+ convTok.getMetadata.put(s"nlpcraft:relation:indexes", refs.map(_.getIndex).asJava)
+ }
+
+ case _ => // No-op.
+ }
+ }
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
index 74e0345..0201315 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
@@ -150,6 +150,21 @@
assertEquals(expResp, res.getResult.get)
}
+ /**
+ *
+ * @param req
+ * @param resExtractor
+ * @param expResp
+ * @tparam T
+ */
+ protected def checkResult[T](req: String, resExtractor: String => T, expResp: T): Unit = {
+ val res = getClient.ask(req)
+
+ assertTrue(res.isOk, s"Unexpected result, error=${res.getResultError.orElse(null)}")
+ assertTrue(res.getResult.isPresent)
+ assertEquals(expResp, resExtractor(res.getResult.get))
+ }
+
final protected def getClient: NCTestClient = {
if (cli == null)
throw new IllegalStateException("Client is not started.")
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCLimitSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCLimitSpec.scala
new file mode 100644
index 0000000..8b9140f
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCLimitSpec.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.models.stm.indexes
+
+import org.apache.nlpcraft.model.{NCIntent, NCIntentMatch, NCResult, _}
+import org.apache.nlpcraft.models.stm.indexes.NCSpecModelAdapter.mapper
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
+import org.junit.jupiter.api.Test
+
+import java.util.{List => JList}
+import scala.jdk.CollectionConverters.ListHasAsScala
+import scala.language.implicitConversions
+
+case class NCLimitSpecModelData(note: String, indexes: Seq[Int])
+
+class NCLimitSpecModel extends NCSpecModelAdapter {
+ @NCIntent("intent=limit term(limit)~{tok_id() == 'nlpcraft:limit'} term(elem)~{has(tok_groups(), 'G')}")
+ private def onLimit(ctx: NCIntentMatch, @NCIntentTerm("limit") limit: NCToken): NCResult =
+ NCResult.json(
+ mapper.writeValueAsString(
+ NCLimitSpecModelData(
+ note = limit.meta[String]("nlpcraft:limit:note"),
+ indexes = limit.meta[JList[Int]]("nlpcraft:limit:indexes").asScala.toSeq
+ )
+ )
+ )
+}
+
+@NCTestEnvironment(model = classOf[NCLimitSpecModel], startClient = true)
+class NCLimitSpec extends NCTestContext {
+ private def extract(s: String): NCLimitSpecModelData = mapper.readValue(s, classOf[NCLimitSpecModelData])
+
+ @Test
+ private[stm] def test(): Unit = {
+ checkResult(
+ "top 23 a a",
+ extract,
+ NCLimitSpecModelData(note = "A", indexes = Seq(1))
+ )
+ checkResult(
+ "test test b b",
+ extract,
+ NCLimitSpecModelData(note = "B", indexes = Seq(2))
+ )
+ }
+}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCRelationSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCRelationSpec.scala
new file mode 100644
index 0000000..c1d42c5
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCRelationSpec.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.models.stm.indexes
+
+import org.apache.nlpcraft.model.{NCIntent, NCIntentMatch, NCResult, _}
+import org.apache.nlpcraft.models.stm.indexes.NCSpecModelAdapter.mapper
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
+import org.junit.jupiter.api.Test
+
+import java.util.{List => JList}
+import scala.jdk.CollectionConverters.ListHasAsScala
+import scala.language.implicitConversions
+
+case class NCRelationSpecModelData(note: String, indexes: Seq[Int])
+
+class NCRelationSpecModel extends NCSpecModelAdapter {
+ @NCIntent("intent=rel term(rel)~{tok_id() == 'nlpcraft:relation'} term(elem)~{has(tok_groups(), 'G')}*")
+ private def onRelation(ctx: NCIntentMatch, @NCIntentTerm("rel") rel: NCToken): NCResult =
+ NCResult.json(
+ mapper.writeValueAsString(
+ NCRelationSpecModelData(
+ note = rel.meta[String]("nlpcraft:relation:note"),
+ indexes = rel.meta[JList[Int]]("nlpcraft:relation:indexes").asScala.toSeq
+ )
+ )
+ )
+}
+
+@NCTestEnvironment(model = classOf[NCRelationSpecModel], startClient = true)
+class NCRelationSpec extends NCTestContext {
+ private def extract(s: String): NCRelationSpecModelData = mapper.readValue(s, classOf[NCRelationSpecModelData])
+
+ @Test
+ private[stm] def test(): Unit = {
+ checkResult(
+ "compare a a and a a",
+ extract,
+ NCRelationSpecModelData(note = "A", indexes = Seq(1, 3))
+ )
+ checkResult(
+ "b b and b b",
+ extract,
+ NCRelationSpecModelData(note = "B", indexes = Seq(0, 2))
+ )
+ }
+}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSortSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSortSpec.scala
new file mode 100644
index 0000000..ed47ade
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSortSpec.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.models.stm.indexes
+
+import org.apache.nlpcraft.model.{NCIntent, NCIntentMatch, NCResult, _}
+import org.apache.nlpcraft.models.stm.indexes.NCSpecModelAdapter.mapper
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
+import org.junit.jupiter.api.Test
+
+import java.util.{List => JList}
+import scala.jdk.CollectionConverters.ListHasAsScala
+import scala.language.implicitConversions
+
+object NCSortSpecModelData {
+ private def nvl[T](list: JList[T]): Seq[T] = if (list == null) Seq.empty else list.asScala.toSeq
+
+ def apply(subjnotes: JList[String], subjindexes: JList[Int], bynotes: JList[String], byindexes: JList[Int]):
+ NCSortSpecModelData = new
+ NCSortSpecModelData(
+ subjnotes = nvl(subjnotes),
+ subjindexes = nvl(subjindexes),
+ bynotes = nvl(bynotes),
+ byindexes = nvl(byindexes)
+ )
+}
+
+case class NCSortSpecModelData(
+ subjnotes: Seq[String] = Seq.empty,
+ subjindexes: Seq[Int] = Seq.empty,
+ bynotes: Seq[String] = Seq.empty,
+ byindexes: Seq[Int] = Seq.empty
+)
+
+class NCSortSpecModel extends NCSpecModelAdapter {
+ @NCIntent("intent=onBySort term(sort)~{tok_id() == 'nlpcraft:sort'} term(elem)~{has(tok_groups(), 'G')}")
+ private def onBySort(ctx: NCIntentMatch, @NCIntentTerm("sort") sort: NCToken): NCResult =
+ NCResult.json(
+ mapper.writeValueAsString(
+ NCSortSpecModelData(
+ subjnotes = sort.meta[JList[String]]("nlpcraft:sort:subjnotes"),
+ subjindexes = sort.meta[JList[Int]]("nlpcraft:sort:subjindexes"),
+ bynotes = sort.meta[JList[String]]("nlpcraft:sort:bynotes"),
+ byindexes = sort.meta[JList[Int]]("nlpcraft:sort:byindexes")
+ )
+ )
+ )
+}
+
+@NCTestEnvironment(model = classOf[NCSortSpecModel], startClient = true)
+class NCSortSpec extends NCTestContext {
+ private def extract(s: String): NCSortSpecModelData = mapper.readValue(s, classOf[NCSortSpecModelData])
+
+ @Test
+ private[stm] def test(): Unit = {
+ checkResult(
+ "test test sort by a a",
+ extract,
+ NCSortSpecModelData(bynotes = Seq("A"), byindexes = Seq(3))
+ )
+ checkResult(
+ "test b b",
+ extract,
+ NCSortSpecModelData(bynotes = Seq("B"), byindexes = Seq(1))
+ )
+ checkResult(
+ "test test sort a a by a a",
+ extract,
+ NCSortSpecModelData(subjnotes = Seq("A"), subjindexes = Seq(3), bynotes = Seq("A"), byindexes = Seq(5))
+ )
+
+// checkResult(
+// "test test sort a a, a a by a a, a a",
+// extract,
+// NCSortSpecModelData(subjnotes = Seq("A"), subjindexes = Seq(2, 3), bynotes = Seq("A"), byindexes = Seq(5, 6))
+// )
+ }
+}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSpecModelAdapter.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSpecModelAdapter.scala
new file mode 100644
index 0000000..7d0c5de
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/indexes/NCSpecModelAdapter.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.models.stm.indexes
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import org.apache.nlpcraft.model.{NCElement, NCModelAdapter}
+
+import java.util
+import java.util.Collections
+import scala.jdk.CollectionConverters.{SeqHasAsJava, SetHasAsJava}
+
+object NCSpecModelAdapter {
+ val mapper = new ObjectMapper()
+
+ mapper.registerModule(new DefaultScalaModule())
+}
+
+class NCSpecModelAdapter extends NCModelAdapter("nlpcraft.stm.idxs.test", "STM Indexes Test Model", "1.0") {
+ override def getElements: util.Set[NCElement] =
+ Set(
+ mkElement("A", "G", "a a"),
+ mkElement("B", "G", "b b")
+ ).asJava
+
+ private def mkElement(id: String, group: String, syns: String*): NCElement =
+ new NCElement {
+ override def getId: String = id
+ override def getSynonyms: util.List[String] = {
+ val seq: Seq[String] = syns
+
+ seq.asJava
+ }
+ override def getGroups: util.List[String] = Collections.singletonList(group)
+ }
+
+
+}
