WIP.
diff --git a/_includes/left-side-menu.html b/_includes/left-side-menu.html
index 0516699..8d51d97 100644
--- a/_includes/left-side-menu.html
+++ b/_includes/left-side-menu.html
@@ -38,33 +38,53 @@
<a href="/first-example.html">First Example</a>
{% endif %}
</li>
+ <li>
+ {% if page.id == api-components" %}
+ <a class="active" href="/api-components.html">API Components</a>
+ {% else %}
+ <a href="/api-components.html">API Components</a>
+ {% endif %}
+ </li>
<li class="side-nav-title">Built-in Components</li>
<li>
- {% if page.id == "semantic" %}
- <a class="active" href="/semantic.html">Semantic Parser</a>
+ {% if page.id == "built-in-overview" %}
+ <a class="active" href="/built-in-overview.html">Overview</a>
{% else %}
- <a href="/semantic.html">Semantic Parser</a>
+ <a href="/built-in-overview.html">Overview</a>
{% endif %}
</li>
<li>
- {% if page.id == "built-in-components" %}
- <a class="active" href="/built-in-components.html">Built-in Components</a>
+ {% if page.id == "built-in-token-parser" %}
+ <a class="active" href="/built-in-token-parser.html">Token parsers</a>
{% else %}
- <a href="/built-in-components.html">Built-in Components</a>
+ <a href="/built-in-token-parser.html">Token parsers</a>
+ {% endif %}
+ </li>
+ <li>
+ {% if page.id == "built-in-token-enricher" %}
+ <a class="active" href="/built-in-token-enricher.html">Token enrichers</a>
+ {% else %}
+ <a href="/built-in-token-enricher.html">Token enrichers</a>
+ {% endif %}
+ </li>
+ <li>
+ {% if page.id == "built-in-entity-parser" %}
+ <a class="active" href="/built-in-entity-parser.html">Entity Parsers</a>
+ {% else %}
+ <a href="/built-in-entity-parser.html">Entity Parsers</a>
+ {% endif %}
+ </li>
+ <li>
+ {% if page.id == "built-in-builder" %}
+ <a class="active" href="/built-in-builder.html">Pipeline builder</a>
+ {% else %}
+ <a href="/built-in-builder.html">Pipeline builder</a>
{% endif %}
</li>
<li class="side-nav-title">Developer Guide</li>
<li>
- {% if page.id == api-review" %}
- <a class="active" href="/api-review.html">API Review</a>
- {% else %}
- <a href="/api-review.html">API Review</a>
- {% endif %}
- </li>
-
- <li>
{% if page.id == "custom-components" %}
<a class="active" href="/custom-components.html">Custom Components</a>
{% else %}
diff --git a/api-components.html b/api-components.html
new file mode 100644
index 0000000..19a509f
--- /dev/null
+++ b/api-components.html
@@ -0,0 +1,498 @@
+---
+active_crumb: Docs
+layout: documentation
+id: api-components
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">API Components<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ NLPCraft project is based on two main concepts <code>Data Model</code> and <code>Client</code>
+ which have API representations
+ <a href="apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a> and
+ <a href="apis/latest/org/apache/nlpcraft/NCModelClient.html">NCModelClient</a>.
+ For work with the system you should prepare <a href="apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a> instance
+ which is based on configuration and list of components named <code>Pipeline</code>.
+ After you just communicate with prepared model via client's methods.
+ </p>
+
+ <ul>
+ <li>
+ <code>Data Model</code> is domain specific object which responsible for user input interpretation.
+ </li>
+ <li>
+ <code>Client</code> is object which allows to communicate with the given data model.
+ </li>
+ </ul>
+
+ <p>Typical part of code:</p>
+
+ <pre class="brush: scala, highlight: []">
+ // Initialized prepared domain model.
+ val mdl = new CustomNlpModel()
+
+ // Creates client for given model.
+ val client = new NCModelClient(mdl)
+
+ // Sends text request to model by user ID "userId".
+ val result = client.ask("Some user command", "userId")
+
+ // Clears dialog session for user with ID "userId".
+ client.clearDialog("userId")
+ </pre>
+ </section>
+
+ <section id="model">
+ <h2 class="section-title">Data Model responsibility<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Let's start with terminology and describe the system work workflow.
+ </p>
+
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Term</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>Token</code></td>
+ <td>
+ <code>Token</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a>.
+ It is simple string, part of user input, which split according to some rules,
+ for instance by spaces and some additional conditions, which depends on language and some expectations.
+ So user input "<b>Where is it?</b>" contains four tokens:
+ "<code>Where</code>", "<code>is</code>", "<code>it</code>", "<code>?</code>".
+ Usually <code>tokens</code> are words and punctuation symbols which can also contain some additional
+ information like point of speech etc.
+ <code>Tokens</code> are input for searching the <code>entities</code>.
+ </td>
+ </tr>
+ <tr>
+ <td><code>Entity</code></td>
+ <td>
+ <code>Entity</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntity.html">NCEntity</a>.
+ According to wikipedia, named entity is a real-world object, such as a person, location, organization,
+ product, etc., that can be denoted with a proper name. It can be abstract or have a physical existence.
+ Each <code>entity</code> can contain one or more tokens.
+ <code>Entities</code> are input for searching <code>intents</code> according to <a href="intent-matching.html">Intent matching</a> conditions.
+ </td>
+ </tr>
+ <tr>
+ <td><code>Variant</code></td>
+ <td>
+ <code>Variant</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCVariant.html">NCVariant</a>.
+ It is a list of <code>entities</code>. Potentially, each <code>token</code> can be recognized as
+ different <code>entities</code>,
+ so user input can be processed as set of <code>variants</code>.
+ For example user input "Mercedes" can be processed as two <code>variants</code>,
+ both of them contains single element list of <code>entities</code>: <b>car brand</b> or <b>Spanish female name</b>.
+ When words are not overlapped with different <code>entities</code> there is only one
+ <code>variant</code> detected.
+ </td>
+ </tr>
+ <tr>
+ <td><code>Intent</code></td>
+ <td>
+ <code>Intent</code> is user defined callback and rule, according to which this callback should be called.
+ Rule is most often some template, based on expected set of <code>entities</code> in user input,
+ but it can be more flexible.
+ Parameters extracted from user text input are passed into callback methods.
+ These methods execution results are provided to user as answer on his request.
+ <code>Intent</code> callbacks are methods defined in <code>Data Model</code> class annotated by
+ <code>intent</code> rules via <a href="intent-matching.html">IDL</a>.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+ <p>
+ <code>Data Model</code> must be able to do tree following things:
+ </p>
+
+ <ul>
+ <li>
+ Parse user input text as the <code>tokens</code>.
+ They are input for searching <code>named entities</code>.
+ <code>Tokens</code> parsing components should be included into <a href="#model-pipeline">Model pipeline</a>.
+ </li>
+ <li>
+ Find <code>named entities</code> based on these parsed <code>tokens</code>.
+ They are input for searching <code>intents</code>.
+ <code>Entity</code> parsing components should be included into <a href="#model-pipeline">Model pipeline</a>.
+ </li>
+ <li>
+ Prepare <code>intents</code> with their callbacks methods which contain business logic.
+ These methods should be defined directly in the model class definition or the model should have references on them.
+ It will be described below.
+ </li>
+ </ul>
+
+ <p>
+ As example, let's prepare the system which can call persons from your contact list.
+ Typical commands are: "<b>Please call to John Smith</b>" or "<b>Connect me with Barbara Dillan</b>".
+ For solving this task this model should be able to recognize in user text following entities:
+ <code>command</code> and <code>person</code> to apply this command.
+ </p>
+
+ <p>
+ So, when request "<b>Please call to John Smith</b>" received, our model should be able to:
+ </p>
+
+ <ul>
+ <li>
+ Parse tokens splitting user text input:
+ "<code>please</code>", "<code>call</code>", "<code>to</code>", "<code>john</code>", "<code>smith</code>".
+ </li>
+ <li>
+ Find two named entities:
+ <ul>
+ <li>
+ <code>command</code> by token "<code>call</code>".
+ </li>
+ <li>
+ <code>person</code> by tokens "<code>john</code>" and "<code>smith</code>".
+ </li>
+ </ul>
+ </li>
+ <li>
+ Have prepared intent:
+ <pre class="brush: scala, highlight: [1, 2, 5, 6]">
+ @NCIntent("intent=call term(command)={# == command'} term(person)={# == 'person'}")
+ def onCommand(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("command") command: NCEntity,
+ @NCIntentTerm("person") person: NCEntity
+ ): NCResult = ? // Implement business logic here.
+ </pre>
+
+ <ul>
+ <li>
+ <code>Line 1</code> defines intent <code>call</code> with two conditions.
+ </li>
+ <li>
+ <code>Line 2</code> defines related callback method <code>onCommand()</code>.
+ </li>
+ <li>
+ <code>Lines 4 and 5</code> define two callback method's arguments which are corresponded to
+ <code>call</code> intent terms conditions. You can extract normalized value
+ <code>john smith</code> from the <code>person</code> parameter and use it in the method body
+ for getting his phone number etc.
+ </li>
+ </ul>
+
+ <p>
+ Note that there are a lot of options of defining intents and their callback methods.
+ They can be defined in configuration files, nested classes, etc.
+ Look at the chapter <a href="intent-matching.html">Intent Matching</a> content for get more details.
+ </p>
+ </li>
+ </ul>
+
+ </section>
+
+ <section id="client">
+ <h2 class="section-title">Client responsibility<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Client which is represented as <a href="apis/latest/org/apache/nlpcraft/NCModelClient.html">NCModelClient</a>
+ is necessary for communication with the model. Base client methods are described below.
+ </p>
+
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Method</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>ask()</code></td>
+ <td>
+ Passes user text input to the model and receives back execution
+ <a href="apis/latest/org/apache/nlpcraft/NCResult.html">NCResult</a> or
+ rejection exception if there isn't any triggered intents.
+ <a href="apis/latest/org/apache/nlpcraft/NCResult.html">NCResult</a> is wrapper on
+ callback method execution result with additional information.
+ </td>
+ </tr>
+ <tr>
+ <td><code>debugAsk()</code></td>
+ <td>
+ Passes user text input to the model and receives back callback and its parameters or
+ rejection exception if there isn't any triggered intents.
+ Main difference from <code>ask</code> that triggered intent callback method is not called.
+ This method and this parameter can be useful for tests scenarios.
+ </td>
+ </tr>
+ <tr>
+ <td><code>clearStm()</code></td>
+ <td>
+ Clears STM state. Memory is cleared wholly or with some predicate.
+ Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
+ </td>
+ </tr>
+ <tr>
+ <td><code>clearDialog()</code></td>
+ <td>
+ Clears dialog state. Dialog is cleared wholly or with some predicate.
+ Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
+ </td>
+ </tr>
+ <tr>
+ <td><code>close()</code></td>
+ <td>
+ Closes client. You can't call another client's methods after this method was closed.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+
+ <section id="model-configuration">
+ <h2 class="section-title">Model configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Data Model configuration <a href="apis/latest/org/apache/nlpcraft/NCModelConfig.html">NCModelConfig</a> represents set of model parameter values.
+ Its properties are described below.
+ </p>
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>Id</code>, <code>Name</code> and <code>Version</code></td>
+ <td>
+ Mandatory model properties.
+ </td>
+ </tr>
+ <tr>
+ <td><code>Description</code>, <code>Origin</code></td>
+ <td>
+ Optional model properties.
+ </td>
+ </tr>
+ <tr>
+ <td><code>conversationTimeout</code></td>
+ <td>
+ Timeout of the user's conversation.
+ If user doesn't communicate with the model this time period STM is going to be cleared.
+ Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
+ Mandatory parameter with default value.
+ </td>
+ </tr>
+ <tr>
+ <td><code>conversationDepth</code></td>
+ <td>
+ Maximum supported depth the user's conversation.
+ Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
+ Mandatory parameter with default value.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </section>
+
+ <section id="model-pipeline">
+ <h2 class="section-title">Model pipeline <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Model <code>Pipeline</code> is represented as <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a> and
+ contains following components:
+ </p>
+
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Component</th>
+ <th>Mandatory</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a></td>
+ <td>Mandatory single</td>
+ <td>
+ <code>Token parser</code> should parse user input plain text and split this text
+ into <code>tokens</code> list.
+ NLPCraft provides default English language implementation of token parser.
+ Also, project contains examples for <a href="examples/light_switch_fr.html">French</a> and
+ <a href="examples/light_switch_ru.html">Russia</a> languages token parser implementations.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a></td>
+ <td>Optional list</td>
+ <td>
+ <code>Tokens enricher</code> is a component which allow to add additional properties to prepared tokens,
+ like part of speech, quote, stop-words flags or any other.
+ NLPCraft provides English language default set of token enrichers implementations.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCTokenValidator.html">NCTokenValidator</a></td>
+ <td>Optional list</td>
+ <td>
+ <code>Token validator</code> is a component which allow to inspect prepared tokens and
+ throw an exception to break user input processing.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a></td>
+ <td>Mandatory list</td>
+ <td>
+ <code>Entity parser</code> is a component which allow to find user defined named entities
+ based on prepared tokens as input.
+ NLPCraft provides wrappers for named-entity recognition components of <a href="https://opennlp.apache.org/">Apache OpenNLP</a> and
+ <a href="https://nlp.stanford.edu/">Stanford NLP</a>.
+ Note that at least one entity parser must be defined.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCEntityEnricher.html">NCEntityEnricher</a></td>
+ <td>Optional list</td>
+ <td>
+ <code>Entity enricher</code> is component which allows to add additional properties to prepared entities.
+ Can be useful for extending existing entity enrichers functionality.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCEntityMapper.html">NCEntityMapper</a></td>
+ <td>Optional list</td>
+ <td>
+ <code>Entity mappers</code> is component which allows to map one set of entities into another after the entities
+ were parsed and enriched. Can be useful for building complex parsers based on existing.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCEntityValidator.html">NCEntityValidator</a></td>
+ <td>Optional list</td>
+ <td>
+ <code>Entity validator</code> is a component which allow to inspect prepared entities and
+ throw an exception to break user input processing.
+ </td>
+ </tr>
+ <tr>
+ <td> <a href="apis/latest/org/apache/nlpcraft/NCVariantFilter.html">NCVariantFilter</a></td>
+ <td>Optional single</td>
+ <td>
+ <code>Variant filter</code> is a component which allows filtering detected variants and rejecting undesirable.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+ <p>
+ Below <code>Model</code> creation example.
+ <code>Pipeline</code> is prepared using <code>NCPipelineBuilder</code> class helper.
+ </p>
+
+ <pre class="brush: scala, highlight: []">
+ val pipeline =
+ new NCPipelineBuilder().
+ withTokenParser(new NCFrTokenParser()).
+ withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
+ withTokenEnricher(new NCFrStopWordsTokenEnricher()).
+ withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
+ build
+ val cfg = NCModelConfig("nlpcraft.lightswitch.fr.ex", "LightSwitch Example Model FR", "1.0")
+
+ val mdl = new NCModelAdapter(cfg, pipeline)
+ </pre>
+
+ <p>
+ This flexible system allows to create any pipelines on any language.
+ You can collect NLPCraft predefined components, write your own and easy reuse custom components.
+ </p>
+ </section>
+
+ <section id="model-behavior">
+ <h2 class="section-title">Model behavior overriding<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ There are also several <a href="/apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a>
+ callbacks that you can override to affect model behavior during
+ <a href="/intent-matching.html#model_callbacks">intent matching</a>
+ to perform logging, debugging, statistic or usage collection, explicit update or initialization of
+ conversation context, security audit or validation:
+ </p>
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Method</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>onMatchedIntent()</code></td>
+ <td>
+ Overriding this method allows to reject matched intent and continue matching process.
+ </td>
+ </tr>
+ <tr>
+ <td><code>onResult()</code></td>
+ <td>
+ Overriding this method allows to replace callback method execution result.
+ </td>
+ </tr>
+ <tr>
+ <td><code>onRejection()</code></td>
+ <td>
+ Overriding this method allows to change operation result when rejection occurs.
+ </td>
+ </tr>
+ <tr>
+ <td><code>onError()</code></td>
+ <td>
+ Overriding this method allows to change operation result when any error occurs.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#model">Model responsibility overview</a></li>
+ <li><a href="#client">Client responsibility overview</a></li>
+ <li><a href="#model-configuration">Model configuration</a></li>
+ <li><a href="#model-pipeline">Model pipeline</a></li>
+ <li><a href="#model-behavior">Model behavior overriding</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/api-review.html b/api-review.html
deleted file mode 100644
index 131bc52..0000000
--- a/api-review.html
+++ /dev/null
@@ -1,377 +0,0 @@
----
-active_crumb: Docs
-layout: documentation
-id: api-review
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">API review <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- NLPCraft library is based on two main concepts <code>Data Model</code> and <code>Client</code>
- which have API representations
- <a href="apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a> and
- <a href="apis/latest/org/apache/nlpcraft/NCModelClient.html">NCModelClient</a>.
- For work with the system you should prepare <a href="apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a> instance
- which is based on configuration and list of components named <code>pipeline</code>.
- After you just communicate with prepared model via client's methods.
- </p>
-
- <ul>
- <li>
- <code>Data Model</code> is domain specific object which responsible for user input interpretation.
- </li>
- <li>
- <code>Client</code> is object which allows to communicate with the given data model.
- </li>
- </ul>
-
- <p>Typical part of code:</p>
-
- <pre class="brush: scala, highlight: []">
- // Initialized prepared domain model.
- val mdl = new CustomNlpModel()
-
- // Creates client for given model.
- val client = new NCModelClient(mdl)
-
- // Sends text request to model by user ID "userId".
- val result = client.ask("Some user command", "userId")
-
- // Clears dialog session for user with ID "userId".
- client.clearDialog("userId")
- </pre>
- </section>
-
- <section id="model">
- <h2 class="section-title">Data Model responsibility<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Let's start with terminology and describe the system work workflow.
- </p>
-
- <ul>
- <li>
- <code>Token</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a>.
- It is simple string, part of user input, which split according to some rules,
- for instance by spaces and some additional conditions, which depends on language and some expectations.
- So user input "<b>Where is it?</b>" contains four tokens:
- "<code>Where</code>", "<code>is</code>", "<code>it</code>", "<code>?</code>".
- Usually <code>tokens</code> are words and punctuation symbols which can also contain some additional
- information like point of speech etc.
- <code>Tokens</code> are input for searching the <code>entities</code>.
- </li>
- <li>
- <code>Entity</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntity.html">NCEntity</a>.
- According to wikipedia, named entity is a real-world object, such as a person, location, organization,
- product, etc., that can be denoted with a proper name. It can be abstract or have a physical existence.
- Each <code>entity</code> can contain one or more tokens.
- <code>Entities</code> are input for searching <code>intents</code> according to <a href="intent-matching.html">Intent matching</a> conditions.
- </li>
- <li>
- <code>Variant</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCVariant.html">NCVariant</a>.
- It is a list of <code>entities</code>. Potentially, each <code>token</code> can be recognized as
- different <code>entities</code>,
- so user input can be processed as set of <code>variants</code>.
- For example user input "Mercedes" can be processed as two <code>variants</code>,
- both of them contains single element list of <code>entities</code>: <b>car brand</b> or <b>Spanish female name</b>.
- When words are not overlapped with different <code>entities</code> there is only one
- <code>variant</code> detected.
- </li>
- <li>
- <code>Intent</code> is user defined callback and rule, according to which this callback should be called.
- Rule is most often some template, based on expected set of <code>entities</code> in user input,
- but it can be more flexible.
- Parameters extracted from user text input are passed into callback methods.
- These methods execution results are provided to user as answer on his request.
- <code>Intent</code> callbacks are methods defined in <code>Data Model</code> class annotated by
- <code>intent</code> rules via <a href="intent-matching.html">IDL</a>.
- </li>
- </ul>
-
- <p>
- <code>Data Model</code> must be able to do tree following things:
- </p>
-
- <ul>
- <li>
- Parse user input text as the <code>tokens</code>.
- They are input for searching <code>named entities</code>.
- <code>Tokens</code> parsing components should be included into <a href="#model-pipeline">Model pipeline</a>.
- </li>
- <li>
- Find <code>named entities</code> based on these parsed <code>tokens</code>.
- They are input for searching <code>intents</code>.
- <code>Entity</code> parsing components should be included into <a href="#model-pipeline">Model pipeline</a>.
- </li>
- <li>
- Prepare <code>intents</code> with their callbacks methods which contain business logic.
- These methods should be defined directly in the model class definition or the model should have references on them.
- It will be described below.
- </li>
- </ul>
-
- <p>
- As example, let's prepare the system which can call persons from your contact list.
- Typical commands are: "<b>Please call to John Smith</b>" or "<b>Connect me with Barbara Dillan</b>".
- For solving this task this model should be able to recognize in user text following entities:
- <code>command</code> and <code>person</code> to apply this command.
- </p>
-
- <p>
- So, when request "<b>Please call to John Smith</b>" received, our model should be able to:
- </p>
-
- <ul>
- <li>
- Parse tokens splitting user text input:
- "<code>please</code>", "<code>call</code>", "<code>to</code>", "<code>john</code>", "<code>smith</code>".
- </li>
- <li>
- Find two named entities:
- <ul>
- <li>
- <code>command</code> by token "<code>call</code>".
- </li>
- <li>
- <code>person</code> by tokens "<code>john</code>" and "<code>smith</code>".
- </li>
- </ul>
- </li>
- <li>
- Have prepared intent:
- <pre class="brush: scala, highlight: [1, 2, 5, 6]">
- @NCIntent("intent=call term(command)={# == command'} term(person)={# == 'person'}")
- def onCommand(
- ctx: NCContext,
- im: NCIntentMatch,
- @NCIntentTerm("command") command: NCEntity,
- @NCIntentTerm("person") person: NCEntity
- ): NCResult = ? // Implement business logic here.
- </pre>
-
- <ul>
- <li>
- <code>Line 1</code> defines intent <code>call</code> with two conditions.
- </li>
- <li>
- <code>Line 2</code> defines related callback method <code>onCommand()</code>.
- </li>
- <li>
- <code>Lines 4 and 5</code> define two callback method's arguments which are corresponded to
- <code>call</code> intent terms conditions. You can extract normalized value
- <code>john smith</code> from the <code>person</code> parameter and use it in the method body
- for getting his phone number etc.
- </li>
- </ul>
-
- <p>
- Note that there are a lot of options of defining intents and their callback methods.
- They can be defined in configuration files, nested classes, etc.
- Look at the chapter <a href="intent-matching.html">Intent Matching</a> content for get more details.
- </p>
- </li>
- </ul>
-
- </section>
-
- <section id="client">
- <h2 class="section-title">Client responsibility<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- Client which is represented as <a href="apis/latest/org/apache/nlpcraft/NCModelClient.html">NCModelClient</a>
- is necessary for communication with the model. Base client methods are described below.
- </p>
-
- <ul>
- <li>
- <code>ask()</code> passes user text input to the model and receives back execution
- <a href="apis/latest/org/apache/nlpcraft/NCResult.html">NCResult</a> or
- rejection exception if there isn't any triggered intents.
- <a href="apis/latest/org/apache/nlpcraft/NCResult.html">NCResult</a> is wrapper on
- callback method execution result with additional information.
- </li>
- <li>
- <code>debugAsk()</code> passes user text input to the model and receives back callback and its parameters or
- rejection exception if there isn't any triggered intents.
- Main difference from <code>ask</code> that triggered intent callback method is not called.
- This method and this parameter can be useful for tests scenarios.
- </li>
- <li>
- <code>clearStm()</code> clears STM state. Memory is cleared wholly or with some predicate.
- Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
- </li>
- <li>
- <code>clearDialog()</code> clears dialog state. Dialog is cleared wholly or with some predicate.
- Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
- </li>
- <li>
- <code>close()</code> - Closes client. You can't call another client's methods after this method was closed.
- </li>
- </ul>
- </section>
-
- <section id="model-configuration">
- <h2 class="section-title">Model configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- Data Model configuration <a href="apis/latest/org/apache/nlpcraft/NCModelConfig.html">NCModelConfig</a> represents set of model parameter values.
- Its properties are described below.
- </p>
- <ul>
- <li>
- <code>id</code>, <code>name</code> and <code>version</code> are mandatory model descriptors.
- </li>
- <li>
- <code>description</code> and <code>origin</code> are optional model descriptors.
- </li>
- <li>
- <code>conversationTimeout</code> - timeout of the user's conversation.
- If user doesn't communicate with the model this time period STM is going to be cleared.
- Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
- Mandatory parameter with default value.
- </li>
- <li>
- <code>conversationDepth</code> - maximum supported depth the user's conversation.
- Loot at <a href="short-term-memory.html">Conversation</a> chapter to get more details.
- Mandatory parameter with default value.
- </li>
- </ul>
- </section>
-
- <section id="model-pipeline">
- <h2 class="section-title">Model pipeline <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- Model <code>Pipeline</code> is represented as <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a> and
- contains following components:
- </p>
- <ul>
- <li>
- <code>Token parser</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>.
- Mandatory pipeline component, it is required for parsing plain text, user input, and split this text
- into tokens list. NLPCraft provides default EN implementation of token parser.
- Also, project contain various examples for <a href="examples/light_switch_fr.html">French</a> and
- <a href="examples/light_switch_ru.html">Russia</a> languages.
- </li>
- <li>
- <code>Tokens enrichers</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a> optional list.
- Tokens enricher is component which allows to add additional properties to prepared tokens,
- like part of speech, quote, stop-words flags or any other.
- NLPCraft provides default set of EN tokens enrichers implementations.
- </li>
- <li>
- <code>Tokens validators</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCTokenValidator.html">NCTokenValidator</a> optional list.
- Tokens validator is user defined component, where tokens are inspected and exception can be thrown
- from user code to break user input processing.
- </li>
- <li>
- <code>Entity parsers</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a> mandatory list.
- At least one entity parser must be defined. Having prepared tokens as input,
- each entity parser tries to find user defined named entities.
- NLPCraft provides wrappers for named-entity recognition components of OpenNLP and Stanford libraries.
- </li>
- <li>
- <code>Entity enrichers</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntityEnricher.html">NCEntityEnricher</a> optional list.
- Entity enricher is component which allows to add additional properties to prepared entities.
- Can be useful for extending existing entity enrichers functionality.
- </li>
- <li>
- <code>Entity mappers</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntityMapper.html">NCEntityMapper</a> optional list.
- Entity mapper is component which allows to map one set of entities into another after the entities
- were parsed and enriched. Can be useful for building complex parsers based on existing.
- </li>
- <li>
- <code>Entity validators</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCEntityValidator.html">NCEntityValidator</a> optional list.
- Entities validator is user defined component, where prepared entities are inspected and
- exceptions can be thrown from user code to break user input processing.
- </li>
- <li>
- <code>Variant filter</code> represented as <a href="apis/latest/org/apache/nlpcraft/NCVariantFilter.html">NCVariantFilter</a>.
- Optional component which allows filtering detected variants, rejecting undesirable.
- </li>
- </ul>
-
- <p>
- Below <code>Model</code> creation example.
- <code>Pipeline</code> is prepared using <code>NCPipelineBuilder</code> class helper.
- </p>
-
- <pre class="brush: scala, highlight: []">
- val pipeline =
- new NCPipelineBuilder().
- withTokenParser(new NCFrTokenParser()).
- withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
- withTokenEnricher(new NCFrStopWordsTokenEnricher()).
- withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
- build
- val cfg = NCModelConfig("nlpcraft.lightswitch.fr.ex", "LightSwitch Example Model FR", "1.0")
-
- val mdl = new NCModelAdapter(cfg, pipeline)
- </pre>
-
- <p>
- This flexible system allows to create any pipelines on any language.
- You can collect NLPCraft predefined components, write your own and easy reuse custom components.
- </p>
- </section>
-
- <section id="model-behavior">
- <h2 class="section-title">Model behavior overriding<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- There are also several <a href="/apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a>
- callbacks that you can override to affect model behavior during
- <a href="/intent-matching.html#model_callbacks">intent matching</a>
- to perform logging, debugging, statistic or usage collection, explicit update or initialization of
- conversation context, security audit or validation:
- </p>
- <ul>
- <li>
- Overriding <code>onMatchedIntent()</code> allows to reject matched intent and continue matching process.
- </li>
- <li>
- Overriding <code>onResult()</code> allows to replace callback method execution result.
- </li>
- <li>
- Overriding <code>onRejection()</code> allows to change operation result when rejection occurs.
- </li>
- <li>
- Overriding <code>onError()</code> allows to change operation result when any error occurs.
- </li>
- </ul>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#model">Model responsibility overview</a></li>
- <li><a href="#client">Client responsibility overview</a></li>
- <li><a href="#model-configuration">Model configuration</a></li>
- <li><a href="#model-pipeline">Model pipeline</a></li>
- <li><a href="#model-behavior">Model behavior overriding</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
diff --git a/built-in-builder.html b/built-in-builder.html
new file mode 100644
index 0000000..a8d9420
--- /dev/null
+++ b/built-in-builder.html
@@ -0,0 +1,144 @@
+---
+active_crumb: Docs
+layout: documentation
+id: built-in-overview
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">Pipeline builder<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/NCPipelineBuilder.html">NCPipelineBuilder</a> class
+ is designed for simplifying preparing <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a> instance.
+ It contains a number of methods <code>withSemantic()</code> which allow to prepare pipeline instance based on
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser">NCSemanticEntityParser</a> and configured language.
+ Currently only English language is supported.
+ It also adds following English built-in components into pipeline:
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnStopWordsTokenEnricher.html">NCEnStopWordsTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnSwearWordsTokenEnricher.html">NCEnSwearWordsTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnQuotesTokenEnricher.html">NCEnQuotesTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnDictionaryTokenEnricher.html">NCEnDictionaryTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnBracketsTokenEnricher.html">NCEnBracketsTokenEnricher</a>.
+ </p>
+ </section>
+
+ <section id="examples">
+ <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p><b>Simple example</b>:</p>
+
+ <pre class="brush: scala, highlight: []">
+ val pipeline = new NCPipelineBuilder().withSemantic("en", "lightswitch_model.yaml").build
+ </pre>
+ <ul>
+ <li>
+ It defines pipeline with all default English language components and one semantic entity parser with
+ model defined in <code>lightswitch_model.yaml</code>.
+ </li>
+ </ul>
+
+ <p><b>Example with pipeline configured by built-in components:</b></p>
+
+ <pre class="brush: scala, highlight: [2, 6, 7, 12, 13, 14, 15]">
+ val pipeline =
+ val stanford =
+ val props = new Properties()
+ props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner")
+ new StanfordCoreNLP(props)
+ val tokParser = new NCStanfordNLPTokenParser(stanford)
+ val stemmer = new NCSemanticStemmer():
+ private val ps = new PorterStemmer
+ override def stem(txt: String): String = ps.synchronized { ps.stem(txt) }
+
+ new NCPipelineBuilder().
+ withTokenParser(tokParser).
+ withTokenEnricher(new NCEnStopWordsTokenEnricher()).
+ withEntityParser(new NCStanfordNLPEntityParser(stanford, Set("number"))).
+ build
+ </pre>
+ <ul>
+ <li>
+ <code>Line 2</code> defines configured <code>StanfordCoreNLP</code> class instance.
+ Look at <a href="https://nlp.stanford.edu/">Stanford NLP</a> documentation for more details.
+ </li>
+ <li>
+ <code>Line 6</code> defines token parser <code>NCStanfordNLPTokenParser</code>, pipeline mandatory component.
+ Note that this one instance is used for two places: in pipeline definition on <code>line 12</code> and
+ in <code>NCSemanticEntityParser</code> definition on <code>line 15</code>.
+ </li>
+ <li>
+ <code>Line 7</code> defines simple implementation of semantic stemmer which is necessary part
+ of <code>NCSemanticEntityParser</code>.
+ </li>
+ <li>
+ <code>Line 13</code> defines configured <code>NCEnStopWordsTokenEnricher</code> token enricher.
+ </li>
+ <li>
+ <code>Line 14</code> defines <code>NCStanfordNLPEntityParser</code> entity parser based on Stanford NER
+ configured for number values detection.
+ </li>
+ <li>
+ <code>Line 14</code> defines <code>NCStanfordNLPEntityParser</code> entity parser based on Stanford NER
+ configured for number values detection.
+ </li>
+ <li>
+ <code>Line 15</code> defines pipeline building.
+ </li>
+ </ul>
+
+ <p><b>Example with pipeline configured by custom components:</b></p>
+
+ <pre class="brush: scala, highlight: []">
+ val pipeline =
+ new NCPipelineBuilder().
+ withTokenParser(new NCFrTokenParser()).
+ withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
+ withTokenEnricher(new NCFrStopWordsTokenEnricher()).
+ withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
+ build
+ </pre>
+
+ <ul>
+ <li>
+ There is the pipeline created for work with French Language. All components of this pipeline are custom components.
+ You can get fore information at examples description chapters:
+ <a href="examples/light_switch_fr.html">Light Switch FR</a> and
+ <a href="examples/light_switch_ru.html">Light Switch RU</a>.
+ Note that these custom components are mostly wrappers on existing solutions and
+ should be prepared just once when you start work with new language.
+ </li>
+ </ul>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#examples">Examples</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/built-in-components.html b/built-in-components.html
deleted file mode 100644
index a3cd108..0000000
--- a/built-in-components.html
+++ /dev/null
@@ -1,276 +0,0 @@
----
-active_crumb: Docs
-layout: documentation
-id: built-in-components
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Built-in components <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- Model <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>
- is base model element. It defines chain of components traits which are responsible for sentence processing.
- Some built-in implementations of these traits are described below.
- </p>
-
- <div class="bq info">
- <p><b>Built-in component licenses.</b></p>
- <p>
- All built-in components which are based on <a href="https://nlp.stanford.edu/">Stanford NLP</a> models and classes
- are provided with <a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU General Public License</a>.
- Look at Stanford NLP <a href="https://nlp.stanford.edu/software/">Software</a> page.
- All such components are placed in special project module <code>nlpcraft-stanford</code>.
- All other components are proved with <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License Version 2.0</a> license.
- </p>
- </div>
-
- <ul>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>.
- There are provided two built-in implementations, both of them are for English language.
- <ul>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a>.
- It is token parser implementation which is wrapper on
- <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project tokenizer.
- </li>
- <li>
- <code>NCStanfordNLPTokenParser</code>. It is token parser implementation which is wrapper on
- <a href="https://nlp.stanford.edu/">Stanford NLP</a> project tokenizer.
- </li>
- </ul>
- </li>
-
- <li>
- <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a>.
- There are provided a number of built-in implementations, all of them are for English language.
- <ul>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> -
- this component allows to add <code>lemma</code> and <code>pos</code> values to processed token.
- Look at these links fpr more details: <a href="https://www.wikiwand.com/en/Lemma_(morphology)">Lemma</a> and
- <a href="https://www.wikiwand.com/en/Part-of-speech_tagging">Part of speech</a>.
- Current implementation is based on <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project components.
- Is uses Apache OpenNLP models, which are accessible
- <a href="http://opennlp.sourceforge.net/models-1.5/">here</a> for POS taggers.
- English lemmatization model is accessible <a href="https://raw.githubusercontent.com/richardwilly98/elasticsearch-opennlp-auto-tagging/master/src/main/resources/models/en-lemmatizer.dict">here</a>.
- You can use any models which are compatible with Apache OpenNLP <a href="https://opennlp.apache.org/docs/2.0.0/apidocs/opennlp-tools/opennlp/tools/postag/POSTaggerME.html">POSTaggerME</a> and
- <a href="https://opennlp.apache.org/docs/2.0.0/apidocs/opennlp-tools/opennlp/tools/lemmatizer/DictionaryLemmatizer.html">DictionaryLemmatizer</a> components.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnBracketsTokenEnricher.html">NCEnBracketsTokenEnricher</a> -
- this component allows to add <code>brackets</code> boolean flag to processed token.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnQuotesTokenEnricher.html">NCEnQuotesTokenEnricher</a> -
- this component allows to add <code>quoted</code> boolean flag to processed token.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnDictionaryTokenEnricher.html">NCEnDictionaryTokenEnricher</a> -
- this component allows to add <code>dict</code> boolean flag to processed token.
- Note that it requires already defined <code>lemma</code> token property,
- You can use <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> or any another component which sets
- <code>lemma</code> into the token.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnStopWordsTokenEnricher.html">NCEnStopWordsTokenEnricher</a> -
- this component allows to add <code>stopword</code> boolean flag to processed token.
- It is based on predefined rules for English language, but it can be also extended by custom user word list and excluded list.
- Note that it requires already defined <code>lemma</code> token property,
- You can use <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> or any another component which sets
- <code>lemma</code> into the token.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnSwearWordsTokenEnricher.html">NCEnSwearWordsTokenEnricher</a> -
- this component allows to add <code>swear</code> boolean flag to processed token.
- </li>
- </ul>
- </li>
-
- <li>
- <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>.
- There are provided a number of language independent built-in implementations.
- They use their own models which can be on different languages.
- <ul>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCNLPEntityParser.html">NCNLPEntityParser</a> converts NLP tokens into entities with four mandatory properties:
- <code>nlp:token:text</code>, <code>nlp:token:index</code>, <code>nlp:token:startCharIndex</code> and
- <code>nlp:token:endCharIndex</code>. However, if any other properties were added into
- processed tokens by <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a> components, they will be copied also with names
- prefixed with <code>nlp:token:</code>.
- It is language independent component.
- Note that converted tokens set can be restricted by predicate.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a> is wrapper on <a href="https://opennlp.apache.org/">Apache OpenNLP</a> NER components.
- Look at the supported <b>Name Finder</b> models <a href="https://opennlp.sourceforge.net/models-1.5/">here</a>.
- For example for English language are accessible: <code>Location</code>, <code>Money</code>,
- <code>Person</code>, <code>Organization</code>, <code>Date</code>, <code>Time</code> and <code>Percentage</code>.
- There are also accessible dome models for another languages.
- </li>
- <li>
- <code>NCStanfordNLPEntityParser</code> is wrapper on <a href="https://nlp.stanford.edu/">Stanford NLP</a> NER components.
- For example for English language are accessible: <code>Location</code>, <code>Money</code>,
- <code>Person</code>, <code>Organization</code>, <code>Date</code>, <code>Time</code> and <code>Percent</code>.
- There are also accessible dome models for another languages.
- Look at the detailed information <a href="https://nlp.stanford.edu/software/CRF-NER.shtml">here</a>.
- </li>
- <li>
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a> is entity parser which is based on list of synonyms elements.
- It is very important component which allow to solve a wide range of tasks.
- If you want to use <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
- with not English language, you have to provide custom
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a> and
- <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
- implementations for required language.
- Look at the <a href="examples/light_switch_fr.html">Light Switch FR</a> for more details.
- Separated chapter <a href="semantic.html">Semantic parser</a> is dedicated to its detailed description.
- </li>
- </ul>
- </li>
- </ul>
-
- <p>
- Following pipeline components cannot have build implementation because their logic are depend on concrete user model:
- <a href="apis/latest/org/apache/nlpcraft/NCTokenValidator.html">NCTokenValidator</a>,
- <a href="apis/latest/org/apache/nlpcraft/NCEntityEnricher.html">NCEntityEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/NCEntityValidator.html">NCEntityValidator</a>,
- <a href="apis/latest/org/apache/nlpcraft/NCEntityMapper.html">NCEntityMapper</a> and
- <a href="apis/latest/org/apache/nlpcraft/NCVariantFilter.html">NCVariantFilter</a>.
- </p>
-
- <p>
- <a href="apis/latest/org/apache/nlpcraft/NCPipelineBuilder.html">NCPipelineBuilder</a> class
- is designed for simplifying preparing <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a> instance.
- It contains a number of methods <code>withSemantic()</code> which allow to prepare pipeline instance based on
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser">NCSemanticEntityParser</a> and configured language.
- Currently only one language is supported - English.
- It also adds following English components into pipeline:
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnStopWordsTokenEnricher.html">NCEnStopWordsTokenEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnSwearWordsTokenEnricher.html">NCEnSwearWordsTokenEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnQuotesTokenEnricher.html">NCEnQuotesTokenEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnDictionaryTokenEnricher.html">NCEnDictionaryTokenEnricher</a>,
- <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnBracketsTokenEnricher.html">NCEnBracketsTokenEnricher</a>.
- </p>
- </section>
-
- <section id="examples">
- <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p><b>Simple example</b>:</p>
-
- <pre class="brush: scala, highlight: []">
- val pipeline = new NCPipelineBuilder().withSemantic("en", "lightswitch_model.yaml").build
- </pre>
- <ul>
- <li>
- It defines pipeline with all default English language components and one semantic entity parser with
- model defined in <code>lightswitch_model.yaml</code>.
- </li>
- </ul>
-
- <p><b>Example with pipeline configured by built-in components:</b></p>
-
- <pre class="brush: scala, highlight: [2, 6, 7, 12, 13, 14, 15]">
- val pipeline =
- val stanford =
- val props = new Properties()
- props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner")
- new StanfordCoreNLP(props)
- val tokParser = new NCStanfordNLPTokenParser(stanford)
- val stemmer = new NCSemanticStemmer():
- private val ps = new PorterStemmer
- override def stem(txt: String): String = ps.synchronized { ps.stem(txt) }
-
- new NCPipelineBuilder().
- withTokenParser(tokParser).
- withTokenEnricher(new NCEnStopWordsTokenEnricher()).
- withEntityParser(new NCStanfordNLPEntityParser(stanford, Set("number"))).
- build
- </pre>
- <ul>
- <li>
- <code>Line 2</code> defines configured <code>StanfordCoreNLP</code> class instance.
- Look at <a href="https://nlp.stanford.edu/">Stanford NLP</a> documentation for more details.
- </li>
- <li>
- <code>Line 6</code> defines token parser <code>NCStanfordNLPTokenParser</code>, pipeline mandatory component.
- Note that this one instance is used for two places: in pipeline definition on <code>line 12</code> and
- in <code>NCSemanticEntityParser</code> definition on <code>line 15</code>.
- </li>
- <li>
- <code>Line 7</code> defines simple implementation of semantic stemmer which is necessary part
- of <code>NCSemanticEntityParser</code>.
- </li>
- <li>
- <code>Line 13</code> defines configured <code>NCEnStopWordsTokenEnricher</code> token enricher.
- </li>
- <li>
- <code>Line 14</code> defines <code>NCStanfordNLPEntityParser</code> entity parser based on Stanford NER
- configured for number values detection.
- </li>
- <li>
- <code>Line 14</code> defines <code>NCStanfordNLPEntityParser</code> entity parser based on Stanford NER
- configured for number values detection.
- </li>
- <li>
- <code>Line 15</code> defines pipeline building.
- </li>
- </ul>
-
- <p><b>Example with pipeline configured by custom components:</b></p>
-
- <pre class="brush: scala, highlight: []">
- val pipeline =
- new NCPipelineBuilder().
- withTokenParser(new NCFrTokenParser()).
- withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
- withTokenEnricher(new NCFrStopWordsTokenEnricher()).
- withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
- build
- </pre>
-
- <ul>
- <li>
- There is the pipeline created for work with French Language. All components of this pipeline are custom components.
- You can get fore information at examples description chapters:
- <a href="examples/light_switch_fr.html">Light Switch FR</a> and
- <a href="examples/light_switch_ru.html">Light Switch RU</a>.
- Note that these custom components are mostly wrappers on existing solutions and
- should be prepared just once when you start work with new language.
- </li>
- </ul>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#examples">Examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
diff --git a/built-in-entity-parser.html b/built-in-entity-parser.html
new file mode 100644
index 0000000..400f4a9
--- /dev/null
+++ b/built-in-entity-parser.html
@@ -0,0 +1,753 @@
+---
+active_crumb: Docs
+layout: documentation
+id: built-in-token-enricher
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">Built-in Entity Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>
+ is a component which allow to find user defined named entities
+ based on prepared tokens as input.
+ </p>
+
+ <p>
+ There are proved:
+ </p>
+
+ <ul>
+ <li>
+ Wrappers for <a href="https://opennlp.apache.org/">Apache OpenNLP</a> and
+ <a href="https://nlp.stanford.edu/">Stanford NLP</a> NER components.
+ Their models support English and some another languages for their NER recognition.
+ </li>
+ <li>
+ NLP data wrapper implementation. It is not depends on language.
+ </li>
+ <li>
+ Semantic implementation for English language.
+ </li>
+ </ul>
+ </section>
+
+ <section id="parser-opennlp">
+ <h2 class="section-title">OpenNLP Based Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a> is wrapper on <a href="https://opennlp.apache.org/">Apache OpenNLP</a> NER components.
+ Look at the supported <b>Name Finder</b> models <a href="https://opennlp.sourceforge.net/models-1.5/">here</a>.
+ For example for English language are accessible: <code>Location</code>, <code>Money</code>,
+ <code>Person</code>, <code>Organization</code>, <code>Date</code>, <code>Time</code> and <code>Percentage</code>.
+ There are also accessible dome models for another languages.
+ </p>
+ </section>
+
+ <section id="parser-stanford">
+ <h2 class="section-title">Stanford NLP OpenNLP Based Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <code>NCStanfordNLPEntityParser</code> is wrapper on <a href="https://nlp.stanford.edu/">Stanford NLP</a> NER components.
+ For example for English language are accessible: <code>Location</code>, <code>Money</code>,
+ <code>Person</code>, <code>Organization</code>, <code>Date</code>, <code>Time</code> and <code>Percent</code>.
+ There are also accessible dome models for another languages.
+ Look at the detailed information <a href="https://nlp.stanford.edu/software/CRF-NER.shtml">here</a>.
+ </p>
+ </section>
+
+ <section id="parser-nlp">
+ <h2 class="section-title">NLP Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCNLPEntityParser.html">NCNLPEntityParser</a> converts NLP tokens into entities with four mandatory properties:
+ <code>nlp:token:text</code>, <code>nlp:token:index</code>, <code>nlp:token:startCharIndex</code> and
+ <code>nlp:token:endCharIndex</code>. However, if any other properties were added into
+ processed tokens by <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a> components, they will be copied also with names
+ prefixed with <code>nlp:token:</code>.
+ It is language independent component.
+ Note that converted tokens set can be restricted by predicate.
+ </p>
+ </section>
+
+ <section id="parser-semantic">
+ <h2 class="section-title">Semantic Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Semantic entity parser
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
+ is the implementation of <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>,
+ which in turn is component of the model pipeline <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>.
+ This parser provides simple but very powerful way to find domain specific data in the input text.
+ It defines list of <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a>
+ which are represent <a href="https://en.wikipedia.org/wiki/Named-entity_recognition">Named entities</a>.
+ We will name this list as <code>Semantic Model</code>.
+ </p>
+
+ <p>
+ Let's talk a little bit more about <a href="https://en.wikipedia.org/wiki/Named-entity_recognition">Named entities</a>.
+ </p>
+
+ <section id="parser-semantic-ne">
+ <h3 class="sub-section-title">Named Entities</h3>
+
+ <p>
+ Named entity, also known as a semantic element or a token, is one of the main a components defined by the NLPCraft data model.
+ A named entity is one or more individual words that have a consistent semantic meaning and typically denote a
+ real-world object, such as persons, locations, number, date and time, organizations, products, etc. Such
+ object can be abstract or have a physical existence.
+ </p>
+ <p>
+ For example, in the following sentence: TODO: PIC
+ </p>
+ <figure>
+ <img alt="named entities" class="img-fluid" src="/images/named-entities.png">
+ <figcaption><b>Fig 2.</b> Named Entities</figcaption>
+ </figure>
+ <p>
+ the following named entities can be detected:
+ </p>
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Words</th>
+ <th>Type</th>
+ <th>Normalized Value</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><b>Top 20</b></td>
+ <td><code>user:element:1</code></td>
+ <td>top 20</td>
+ </tr>
+ <tr>
+ <td><b>best pages</b></td>
+ <td><code>user:element:2</code></td>
+ <td>best pages</td>
+ </tr>
+ <tr>
+ <td><b>California USA</b></td>
+ <td><code>stanford:city</code></td>
+ <td>USA, California</td>
+ </tr>
+ <tr>
+ <td><b>last 3 months</b></td>
+ <td><code>stanford:date</code></td>
+ <td>1/1/2021 - 4/1/2021</td>
+ </tr>
+ </tbody>
+ </table>
+ <p>
+ In most cases named entities will have associated <em>normalized value</em>. It is especially important for named entities that have many
+ notational forms such as time and date, currency, geographical locations, etc. For example, <code>New York</code>,
+ <code>New York City</code> and <code>NYC</code> all refer to the same "New York City, NY USA" location which is a standard normalized form.
+ </p>
+ <p>
+ The process of detecting named entities is called Named Entity Recognition (NER). There are many ways of how a certain named entity can be detected: through list of synonyms, by name, rule-based or by using
+ statistical techniques like neural networks with large corpus of predefined data. NLPCraft natively supports synonym-based
+ named entities definition as well as the ability to compose new named entities through powerful <a href="/intent-matching.html">Intent Definition Language</a> (IDL)
+ combining other named entities including named entities from
+ such OpenNLP, or Stanford CoreNLP, look at the <a href="built-in-entity-parser.html">Built-in Entity Parser</a> chapter.
+ </p>
+ <p>
+ Named entities allow you to abstract from basic linguistic forms like nouns and verbs to deal with the higher level semantic
+ abstractions like geographical location or time when you are trying to understand the meaning of the sentence.
+ One of the main goals of named entities is to act as an input ingredients for <a href="/intent-matching.html">intent matching</a>.
+ </p>
+ <div class="bq info">
+ <p>
+ <b>😀 User Input → Named Entities → Parsing Variants → Intent Matcher → Winning Intent 🚀</b>
+ </p>
+ <p>
+ User input is parsed into the list of named entities. That list is then further transformed into one or more
+ parsing variants where each variant represents a particular order and combination of detected named entities.
+ Finally, the list of variants act as an input to intent matching where each variant is matched against every intent
+ in the process of detecting the best matching intent for the original user input.
+ </p>
+ </div>
+ </section>
+
+ <section id="parser-semantic-elements">
+ <h3 class="sub-section-title">Elements</h3>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> represents
+ NER element for its detection un the user input.
+ <p>
+
+ <div class="bq info">
+ <p>
+ <b>Semantic Element <span class="amp">&</span> Named Entity <span class="amp">&</span> Token</b>
+ </p>
+ <p>
+ Terms 'semantic element', 'named entity' and 'token' are used throughout this documentation relatively interchangeably:
+ </p>
+ <dl>
+ <dt>Semantic Element</dt>
+ <dd>
+ Denotes a named entity <em>declared</em> in NLPCraft model.
+ </dd>
+ <dt>Token</dt>
+ <dd>
+ Denotes a semantic element that was <em>detected</em> by NLPCraft in the user input.
+ </dd>
+ <dt>Named Entity</dt>
+ <dd>
+ Denotes a classic term, i.e. one or more individual words that have a
+ consistent semantic meaning and typically define a real-world object.
+ </dd>
+ </dl>
+ </div>
+
+ <p>
+ Each <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a>
+ is presented by <code>id</code>, <code>groups</code>, <code>synonyms</code>, <code>values</code> and <code>properties</code>.
+ <p>
+ <span id="synonyms" class="section-sub-title">Synonyms <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ NLPCraft uses fully deterministic named entity recognition and is not based on statistical approaches that
+ would require pre-existing marked up data sets and extensive training. For each semantic element you can either provide a
+ set of synonyms to match on or specify a piece of code that would be responsible for detecting that named
+ entity (discussed below). A synonym can have one or more individual words. Note that element's ID is its
+ implicit synonym so that even if no additional synonyms are defined at least one synonym always exists. Note
+ also that synonym matching is performed on <em>normalized</em> and <em>stemmatized</em> forms of both
+ a synonym and user input.
+ </p>
+
+ <p>
+ Here's an example of a simple semantic element definition in JSON:
+ </p>
+ <pre class="brush: js, highlight: [6,7,8,9,10,11,12]">
+ ...
+ "elements": [
+ {
+ "id": "transport.vehicle",
+ "description": "Transportation vehicle",
+ "synonyms": [
+ "car",
+ "truck",
+ "light duty truck"
+ "heavy duty truck"
+ "sedan",
+ "coupe"
+ ]
+ }
+ ]
+ ...
+ </pre>
+ <p>
+ While adding multi-word synonyms looks somewhat
+ trivial - in real models, the naive approach can lead to thousands and even tens of thousands of
+ possible synonyms due to words, grammar, and linguistic permutations - which quickly becomes untenable if
+ performed manually.
+ </p>
+ <p>
+ NLPCraft provides an effective tool for a compact synonyms representation. Instead of listing all possible
+ multi-word synonyms one by one you can use combination of following techniques:
+ </p>
+ <ul>
+ <li><a href="#macros">Macros</a></li>
+ <li><a href="#regex">Regular expressions</a></li>
+ <li><a href="#option-groups">Option Groups</a></li>
+ </ul>
+ <p>
+ Each whitespace separated string in the synonym can be either a regular word (like in the above transportation example
+ where it will be matched on using its normalized and stemmatized form) or one of the above expression.
+ </p>
+ <p>
+ Note that this synonyms definition is also used in the following
+ <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> methods:
+ </p>
+ <ul>
+ <li><code>getSynonyms()</code> - gets synonyms to match on.</li>
+ <li><code>getValues()</code> - get values to match on (see <a href="#values">below</a>).</li>
+ </ul>
+ <span id="values" class="section-sub-title">Element Values <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ Semantic element can have an optional set of special synonyms called <em>values</em> or "proper nouns" for this element.
+ Unlike basic synonyms, each value is a pair of a name and a set of standard synonyms by which that value,
+ and ultimately its element, can be recognized in the user input. Note that the value name itself acts as an
+ implicit synonym even when no additional synonyms added for that value.
+ </p>
+ <p>
+ When a semantic element is recognized it is made available to the model's matching logic as an instance of
+ the <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a> interface.
+ This interface has a method
+ <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">getValue()</a> which
+ returns the name of the value, if any, by which
+ that semantic element was recognized. That value name can be further used in intent matching.
+ </p>
+ <p>
+ To understand the importance of the values consider the following changes to our transportation
+ example model:
+ </p>
+ <pre class="brush: js, highlight: [19,20,21,22,23,24,25,26,27,28,29,30]">
+ ...
+ "macros": [
+ {
+ "name": "<TRUCK_TYPE>",
+ "macro": "{light duty|heavy duty|half ton|1/2 ton|3/4 ton|one ton|super duty}"
+ }
+ ]
+ "elements": [
+ {
+ "id": "transport.vehicle",
+ "description": "Transportation vehicle",
+ "synonyms": [
+ "car",
+ "{<TRUCK_TYPE>|_} {pickup|_} truck"
+ "sedan",
+ "coupe"
+ ],
+ "values": [
+ {
+ "value": "mercedes",
+ "synonyms": ["mercedes-ben{z|s}", "mb", "ben{z|s}"]
+ },
+ {
+ "value": "bmw",
+ "synonyms": ["{bimmer|bimer|beemer}", "bayerische motoren werke"]
+ }
+ {
+ "value": "chevrolet",
+ "synonyms": ["chevy"]
+ }
+ ]
+ }
+ ]
+ ...
+ </pre>
+ <p>
+ With that setup <code>transport.vehicle</code> element will be recognized by any of the following input string:
+ </p>
+ <ul>
+ <li><code>car</code></li>
+ <li><code>benz</code> (with value <code>mercedes</code>)</li>
+ <li><code>3/4 ton pickup truck</code></li>
+ <li><code>light duty truck</code></li>
+ <li><code>chevy</code> (with value <code>chevrolet</code>)</li>
+ <li><code>bimmer</code> (with value <code>bmw</code>)</li>
+ <li><code>transport.vehicle</code></li>
+ </ul>
+ <span id="groups" class="section-sub-title">Element Groups <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ Each semantic element always belongs to one or more groups. Semantic element provides its groups via
+ <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">getGroups()</a> method.
+ By default, if element group is not specified, the element ID will act as its default group ID.
+ Group membership is a quick and easy way to organise similar semantic elements together and use this
+ categorization in <a href="/intent-matching.html">IDL</a> intents.
+ </p>
+ <p>
+ Note that the proper grouping of the elements is also necessary for the correct operation of
+ Short-Term-Memory (STM) in the conversational context. Consider a
+ <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a> that
+ represents a previously found semantic element that is stored in the conversation. Such token
+ will be overridden in the conversation by the more <b>recent token</b>
+ from the <b>same group</b> - a critical rule of maintaining the proper conversational context.
+ See
+ <a href="/apis/latest/org/apache/nlpcraft/NCConversation.html">NCConversation</a>
+ for mode details.
+ </p>
+
+ <span id="macros" class="section-sub-title">Macros<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ Listing all possible multi-word synonyms for a given element can be a time-consuming task. Macros
+ together with option groups allow for significant simplification of this task.
+ Macros allow you to give a name to an often used set of words or option groups and reuse it without
+ repeating those words or option groups again and again. A model provides a list of macros via
+ <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">macros</a> method.
+ Each macro has a name in a form of <code><X></code> where <code>X</code>
+ is any string, and a string value. Note that macros can be nested (but not recursive), i.e. macro value can include
+ references to other macros. When macro name <code>X</code> is encountered in the synonym it gets recursively
+ replaced with its value.
+ </p>
+ <p>
+ Here's a code snippet of macro definitions using JSON definition:
+ </p>
+ <pre class="brush: js">
+ "macros": [
+ {
+ "name": "<A>",
+ "macro": "aaa"
+ },
+ {
+ "name": "<B>",
+ "macro": "<A> bbb"
+ },
+ {
+ "name": "<C>",
+ "macro": "<A> bbb {z|w}"
+ }
+ ]
+ </pre>
+ <span id="option-groups" class="section-sub-title">Option Groups <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ Option groups are similar to wildcard patterns that operates on a single word base. One line of
+ option group expands into one or more individual synonyms. Option groups is the key mechanism for shortened
+ synonyms notation. The following examples demonstrate how to use option groups.
+ </p>
+ <p>
+ Consider the following macros defined below (note that macros <code><B></code> and <code><C></code>
+ are nested):
+ </p>
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Value</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code><A></code></td>
+ <td><code>aaa</code></td>
+ </tr>
+ <tr>
+ <td><code><B></code></td>
+ <td><code><A> bbb</code></td>
+ </tr>
+ <tr>
+ <td><code><C></code></td>
+ <td><code><A> bbb {z|w}</code></td>
+ </tr>
+ </tbody>
+ </table>
+ <p>
+ Then the following option group expansions will occur in these examples:
+ </p>
+ <table class="gradient-table">
+ <thead>
+ <tr>
+ <th>Synonym</th>
+ <th>Synonym Expansions</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code><A> {b|_} c</code></td>
+ <td>
+ <code>"aaa b c"</code><br>
+ <code>"aaa c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code><A> {b|a}[1,2] c</code></td>
+ <td>
+ <code>"aaa b c"</code><br>
+ <code>"aaa b b c"</code><br>
+ <code>"aaa a c"</code><br>
+ <code>"aaa a a c"</code><br>
+ <code>"aaa c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <code><B> {b|_} c</code><br>
+ or<br>
+ <code><B> {b}[0,1] c</code>
+ </td>
+ <td>
+ <code>"aaa bbb b c"</code><br>
+ <code>"aaa bbb c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>{b|\{\_\}}</code></td>
+ <td>
+ <code>"b"</code><br>
+ <code>"b {_}"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>a {b|_}. c</code></td>
+ <td>
+ <code>"a b. c"</code><br>
+ <code>"a . c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>a .{b, |_}. c</code></td>
+ <td>
+ <code>"a .b, . c"</code><br>
+ <code>"a .. c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>
+ {% raw %}a {{b|c}|_}.{% endraw %}</code></td>
+ <td>
+ <code>"a ."</code><br>
+ <code>"a b."</code><br>
+ <code>"a c."</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>a {% raw %}{{{<C>}}|{_}}{% endraw %} c</code></td>
+ <td>
+ <code>"a aaa bbb z c"</code><br>
+ <code>"a aaa bbb w c"</code><br>
+ <code>"a c"</code>
+ </td>
+ </tr>
+ <tr>
+ <td><code>{% raw %}{{{a}}} {b||_|{{_}}||_}{% endraw %}</code></td>
+ <td>
+ <code>"a b"</code><br>
+ <code>"a"</code>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <p>
+ Specifically:
+ </p>
+ <ul>
+ <li><code>{A|B}</code> denotes either <code>A</code> or <code>B</code>.</li>
+ <li>
+ <code>{A|B|_}</code> denotes either <code>A</code> or <code>B</code> or nothing.
+ <ul>
+ <li>Symbol <code>_</code> cam appear anywhere in the list of options, i.e. <code>{A|B|_}</code> is equal to <code>{A|_|B}</code>.</li>
+ </ul>
+ </li>
+ <li>
+ <code>{C}[x,y]</code> denotes an option group with quantifier, i.e. group <code>C</code> appearing from <code>x</code> to <code>y</code> times inclusive.
+ <ul>
+ <li>For example, <code>{C}[1,3]</code> is the same as <code>{C|C C|C C C}</code> notation.</li>
+ <li>Note that <code>{C|_}</code> is equal to <code>{C}[0,1]</code></li>
+ </ul>
+ </li>
+ <li>Excessive curly brackets are ignored, when safe to do so.</li>
+ <li>Macros cannot be recursive but can be nested.</li>
+ <li>Option groups can be nested.</li>
+ <li>
+ <code>'\'</code> (backslash) can be used to escape <code>'{'</code>, <code>'}'</code>, <code>'|'</code> and
+ <code>'_'</code> special symbols used by the option groups.
+ </li>
+ <li>Excessive whitespaces are trimmed when expanding option groups.</li>
+ </ul>
+ <p>
+ We can rewrite our transportation semantic element in a more efficient way using macros and option groups.
+ Even though the actual length of definition hasn't changed much it now auto-generates many dozens of synonyms
+ we would have to write out manually otherwise:
+ </p>
+ <pre class="brush: js, highlight: [4,5,14]">
+ ...
+ "macros": [
+ {
+ "name": "<TRUCK_TYPE>",
+ "macro": "{ {light|super|heavy|medium} duty|half ton|1/2 ton|3/4 ton|one ton}"
+ }
+ ]
+ "elements": [
+ {
+ "id": "transport.vehicle",
+ "description": "Transportation vehicle",
+ "synonyms": [
+ "car",
+ "{<TRUCK_TYPE>|_} {pickup|_} truck"
+ "sedan",
+ "coupe"
+ ]
+ }
+ ]
+ ...
+ </pre>
+ <span id="regex" class="section-sub-title">Regular Expressions <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
+ <p>
+ Any individual synonym word that starts and ends with <code>//</code> (two forward slashes) is
+ considered to be Java regular expression as defined in <code>java.util.regex.Pattern</code>. Note that
+ regular expression can only span a single word, i.e. only individual words from the user input will be
+ matched against given regular expression and no whitespaces are allowed within regular expression. Note
+ also that option group special symbols <code>{</code>, <code>}</code>,
+ <code>|</code> and <code>_</code> have to be escaped in the regular expression using <code>\</code>
+ (backslash).
+ </p>
+ <p>
+ For example, the following synonym:
+ </p>
+ <pre class="brush: js">
+ "synonyms": [
+ "{foo|//[bar].+//}}"
+ ]
+ </pre>
+ <p>
+ will match word <code>foo</code> or any other strings that start with <code>bar</code> as long as
+ this string doesn't contain whitespaces.
+ </p>
+ <div class="bq info">
+ <b>Regular Expressions Performance</b>
+ <p>
+ It's important to note that regular expressions can significantly affect the performance of the
+ NLPCraft processing if used uncontrolled. Use it with caution and test the performance
+ of your model to ensure it meets your requirements.
+ </p>
+ </div>
+ </section>
+
+ <section id="parser-semantic-examples">
+ <h3 class="sub-section-title">Examples</h3>
+
+ <p>
+ The following example shows how to build model programmatically.
+ </p>
+
+ <pre class="brush: scala, highlight: [3, 5, 10]">
+ val mdl = new NCModelAdapter(
+ NCModelConfig("test.id", "Test Model", "1.0"),
+ new NCPipelineBuilder().withSemantic(
+ "en",
+ Map(
+ "<OF>" -> "{of|for|per}",
+ "<CUR>" -> "current|present|now|local}",
+ "<TIME>" -> "{time <OF> day|day time|date|time|moment|datetime|hour|o'clock|clock|date time|date and time|time and date}",
+ )
+ List(
+ new NCSemanticElement():
+ override def getId: String = "time"
+ override def getSynonyms: Set[String] = Set("{<CUR>|_} <TIME>", "what <TIME> {is it now|now|is it|_}" )
+ )
+ ).build
+ )
+ </pre>
+ <ul>
+ <li>
+ <code>Line 5</code> shows <code>macro</code> parameter usage.
+ </li>
+ <li>
+ <code>Line 10</code> shows <code>macro</code> list of <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> parameter usage.
+ </li>
+ <li>
+ Note that usage <code>withSemantic()</code> method which represented on <code>line 3</code> i optional.
+ You can add <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCNLPEntityParser.html">NCNLPEntityParser</a>
+ as usual <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>
+ when you define your <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>.
+ </li>
+ </ul>
+
+ <p>
+ The following examples is based on YAML semantic elements representation.
+ </p>
+
+ <pre class="brush: js, highlight: []">
+ macros:
+ "<OF>": "{of|for|per}"
+ "<CUR>": "{current|present|now|local}"
+ "<TIME>": "{time <OF> day|day time|date|time|moment|datetime|hour|o'clock|clock|date time|date and time|time and date}"
+ elements:
+ - id: "x:time"
+ description: "Date and/or time token indicator."
+ synonyms:
+ - "{<CUR>|_} <TIME>"
+ - "what <TIME> {is it now|now|is it|_}"
+ </pre>
+ <ul>
+ <li>
+ Same macros and the same element as in previous example are defined here in
+ <code>time_model.yaml</code> YAML file.
+ </li>
+ </ul>
+ <pre class="brush: scala, highlight: [3]">
+ val mdl = new NCModelAdapter(
+ NCModelConfig("test.id", "Test Model", "1.0"),
+ new NCPipelineBuilder().withSemantic("en", "time_model.yaml").build
+ )
+ </pre>
+ <ul>
+ <li>
+ <code>Line 3</code> makes semantic model which defined in <code>time_model.yaml</code> YAML file.
+ </li>
+ </ul>
+
+ <p>
+ If you want to use <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
+ with not English language, you have to provide custom
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a> and
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
+ implementations for required language. Look at the <a href="examples/light_switch_fr.html">Light Switch FR</a>
+ for more details.
+ </p>
+
+ <pre class="brush: scala, highlight: [4, 7, 8]">
+ package demo
+
+ import opennlp.tools.stemmer.snowball.SnowballStemmer
+ import demo.nlp.token.parser.NCFrTokenParser
+ import org.apache.nlpcraft.nlp.parsers.*
+
+ class NCFrSemanticEntityParser(src: String) extends NCSemanticEntityParser(
+ new NCSemanticStemmer:
+ private val stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.FRENCH)
+ override def stem(txt: String): String = stemmer.synchronized { stemmer.stem(txt.toLowerCase).toString }
+ ,
+ new NCFrTokenParser(),
+ mdlSrcOpt = Option(src)
+ )
+ </pre>
+ <ul>
+ <li>
+ <code>Line 4</code> includes <code>NCFrTokenParser</code> import.
+ Its custom <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
+ implementation for French language, described here: <a href="examples/light_switch_fr.html">Light Switch FR</a>.
+ </li>
+ <li>
+ <code>Line 8</code> defines custom <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a>
+ implementation for French language.
+ </li>
+ <li>
+ As you can see, <code>NCFrSemanticEntityParser</code> is very simple extension of
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
+ base class, look at <code>line 7</code>.
+ </li>
+ </ul>
+ </section>
+ <section id="parser-semantic-extending">
+ <h3 class="sub-section-title">Languages Extending</h3>
+
+ <p>
+ If you want to use <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
+ with not English language, you have to provide custom
+ <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a> and
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
+ implementations for required language.
+ Look at the <a href="examples/light_switch_fr.html">Light Switch FR</a> for more details.
+ </p>
+ </section>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#parser-opennlp">OpenNLP Based Parser</a></li>
+ <li><a href="#parser-stanford">Stanford NLP Based Entity</a></li>
+ <li><a href="#parser-nlp">NLP Parser</a></li>
+ <li><a href="#parser-semantic">Semantic Parser</a></li>
+ <li><a href="#parser-semantic-ne">Semantic Parser Named Entities</a></li>
+ <li><a href="#parser-semantic-elements">Semantic Parser Elements</a></li>
+ <li><a href="#parser-semantic-examples">Semantic Parser Examples</a></li>
+ <li><a href="#parser-semantic-extending">SemanticParser Languages Extending</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/built-in-overview.html b/built-in-overview.html
new file mode 100644
index 0000000..a13cf5c
--- /dev/null
+++ b/built-in-overview.html
@@ -0,0 +1,74 @@
+---
+active_crumb: Docs
+layout: documentation
+id: built-in-overview
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">Built-in Components Overview<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Model <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a> contains
+ chain of components traits which are responsible for sentence processing.
+ Some built-in implementations of these traits are described below.
+ </p>
+
+ <p>There are two kinds of pipeline components:</p>
+ <ul>
+ <li>
+ Following pipeline components have built-in implementations and described in related sections:
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>,
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>.
+ </li>
+ <li>
+ Following pipeline components cannot have build implementation because their logic are depend on concrete user model:
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenValidator.html">NCTokenValidator</a>,
+ <a href="apis/latest/org/apache/nlpcraft/NCEntityEnricher.html">NCEntityEnricher</a>,
+ <a href="apis/latest/org/apache/nlpcraft/NCEntityValidator.html">NCEntityValidator</a>,
+ <a href="apis/latest/org/apache/nlpcraft/NCEntityMapper.html">NCEntityMapper</a> and
+ <a href="apis/latest/org/apache/nlpcraft/NCVariantFilter.html">NCVariantFilter</a>.
+ </li>
+ </ul>
+
+ <div class="bq info">
+ <p><b>Built-in component licenses.</b></p>
+ <p>
+ All built-in components which are based on <a href="https://nlp.stanford.edu/">Stanford NLP</a> models and classes
+ are provided with <a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU General Public License</a>.
+ Look at Stanford NLP <a href="https://nlp.stanford.edu/software/">Software</a> page.
+ All such components are placed in special project module <code>nlpcraft-stanford</code>.
+ All other components are proved with <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License Version 2.0</a> license.
+ </p>
+ </div>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/built-in-token-enricher.html b/built-in-token-enricher.html
new file mode 100644
index 0000000..ddb6929
--- /dev/null
+++ b/built-in-token-enricher.html
@@ -0,0 +1,115 @@
+---
+active_crumb: Docs
+layout: documentation
+id: built-in-token-enricher
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">Built-in Token Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenEnricher.html">NCTokenEnricher</a>
+ is a component which allows to add additional properties to prepared tokens,
+ like part of speech, quote, stop-words flags or any other.
+ NLPCraft provides English language default set of token enrichers implementations.
+ </p>
+ </section>
+
+ <section id="enricher-opennlp-lemmapos">
+ <h2 class="section-title">Lemma And POS Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> -
+ this component allows to add <code>lemma</code> and <code>pos</code> values to processed token.
+ Look at these links fpr more details: <a href="https://www.wikiwand.com/en/Lemma_(morphology)">Lemma</a> and
+ <a href="https://www.wikiwand.com/en/Part-of-speech_tagging">Part of speech</a>.
+ Current implementation is based on <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project components.
+ Is uses Apache OpenNLP models, which are accessible
+ <a href="http://opennlp.sourceforge.net/models-1.5/">here</a> for POS taggers.
+ English lemmatization model is accessible <a href="https://raw.githubusercontent.com/richardwilly98/elasticsearch-opennlp-auto-tagging/master/src/main/resources/models/en-lemmatizer.dict">here</a>.
+ You can use any models which are compatible with Apache OpenNLP <a href="https://opennlp.apache.org/docs/2.0.0/apidocs/opennlp-tools/opennlp/tools/postag/POSTaggerME.html">POSTaggerME</a> and
+ <a href="https://opennlp.apache.org/docs/2.0.0/apidocs/opennlp-tools/opennlp/tools/lemmatizer/DictionaryLemmatizer.html">DictionaryLemmatizer</a> components.
+ </p>
+ </section>
+ <section id="enricher-opennlp-bracket">
+ <h2 class="section-title">Brackets Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnBracketsTokenEnricher.html">NCEnBracketsTokenEnricher</a> -
+ this component allows to add <code>brackets</code> boolean flag to processed token.
+ </p>
+ </section>
+ <section id="enricher-opennlp-quotes">
+ <h2 class="section-title">Quotes Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnQuotesTokenEnricher.html">NCEnQuotesTokenEnricher</a> -
+ this component allows to add <code>quoted</code> boolean flag to processed token.
+ </p>
+ </section>
+ <section id="enricher-opennlp-dict">
+ <h2 class="section-title">Dictionary Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnDictionaryTokenEnricher.html">NCEnDictionaryTokenEnricher</a> -
+ this component allows to add <code>dict</code> boolean flag to processed token.
+ Note that it requires already defined <code>lemma</code> token property,
+ You can use <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> or any another component which sets
+ <code>lemma</code> into the token.
+ </p>
+ </section>
+ <section id="enricher-opennlp-stopword">
+ <h2 class="section-title">Stop-words Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnStopWordsTokenEnricher.html">NCEnStopWordsTokenEnricher</a> -
+ this component allows to add <code>stopword</code> boolean flag to processed token.
+ It is based on predefined rules for English language, but it can be also extended by custom user word list and excluded list.
+ Note that it requires already defined <code>lemma</code> token property,
+ You can use <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCOpenNLPLemmaPosTokenEnricher.html">NCOpenNLPLemmaPosTokenEnricher</a> or any another component which sets
+ <code>lemma</code> into the token.
+ </p>
+ </section>
+ <section id="enricher-opennlp-swearword">
+ <h2 class="section-title">Swear-words Enricher<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/nlp/enrichers/NCEnSwearWordsTokenEnricher.html">NCEnSwearWordsTokenEnricher</a> -
+ this component allows to add <code>swear</code> boolean flag to processed token.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#enricher-opennlp-lemmapos">Lemma And POS Enricher</a></li>
+ <li><a href="#enricher-opennlp-bracket">Brackets Enricher</a></li>
+ <li><a href="#enricher-opennlp-quotes">Quotes Enricher</a></li>
+ <li><a href="#enricher-opennlp-dict">Dictionary Enricher</a></li>
+ <li><a href="#enricher-opennlp-stopword">Stop-words Enricher</a></li>
+ <li><a href="#enricher-opennlp-swearword">Swear-words Enricher</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/built-in-token-parser.html b/built-in-token-parser.html
new file mode 100644
index 0000000..a961125
--- /dev/null
+++ b/built-in-token-parser.html
@@ -0,0 +1,94 @@
+---
+active_crumb: Docs
+layout: documentation
+id: built-in-token-parser
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column">
+ <section id="overview">
+ <h2 class="section-title">Built-in Token Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
+ component implementation should parse user input plain text and split this text
+ into <code>tokens</code> list.
+
+ NLPCraft provides default English language implementation of token parser.
+
+ Also, project contains examples for <a href="examples/light_switch_fr.html">French</a> and
+ <a href="examples/light_switch_ru.html">Russia</a> languages token parser implementations.
+ </p>
+
+ </section>
+
+ <section id="parser-opennlp">
+ <h2 class="section-title">Apache OpenNLP Based Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ There is <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCOpenNLPTokenParser.html">NCOpenNLPTokenParser</a> implementation.
+
+ It is token parser implementation which is wrapper on
+ <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project tokenizer.
+ </p>
+ </section>
+
+ <section id="parser-stanford">
+ <h2 class="section-title">Stanford NLP Based Parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ There is <a href="NCStanfordNLPTokenParser.html">NCStanfordNLPTokenParser</a> implementation.
+
+ It is token parser implementation which is wrapper on
+ <a href="https://nlp.stanford.edu/">Stanford NLP</a> project tokenizer.
+ </p>
+ </section>
+ <section id="remarks">
+ <h2 class="section-title">Remarks<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ There are two built-in token parsers added for one English language because they have some difference
+ in their algorithm and can provide different list of tokens for same user text input.
+ Some built-in components are required token parser instance as their parameter.
+
+ </p>
+ <ul>
+ <li>
+ If you use <a href="https://opennlp.apache.org/">Apache OpenNLP</a> based components
+ you should use <a href="#parser-opennlp">Apache OpenNLP based parser</a> in your model pipeline.
+ </li>
+ <li>
+ If you use <a href="https://nlp.stanford.edu/">Stanford NLP</a> based components
+ you should use <a href="#parser-stanford">Stanford based parser</a> in your model pipeline.
+ </li>
+ </ul>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#parser-opennlp">Apache OpenNLP Based Parser</a></li>
+ <li><a href="#parser-stanford">Stanford NLP Based Parser</a></li>
+ <li><a href="#remarks">Remarks</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
diff --git a/custom-components.html b/custom-components.html
index 1905349..fb6d403 100644
--- a/custom-components.html
+++ b/custom-components.html
@@ -26,14 +26,8 @@
<h2 class="section-title">Custom components <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- NLPCraft provides a numeric of useful built-in components for English language.
- You can use them to prepare <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>
- for your <a href="apis/latest/org/apache/nlpcraft/NCModel.html">NCModel</a>.
- You also can use provided wrappers on <a href="https://opennlp.apache.org/">Apache OpenNLP</a> and
- <a href="https://nlp.stanford.edu/">Stanford NLP</a> projects NER components.
- Their models work with English and some another languages.
- </p>
- <p>
+ NLPCraft provides a numeric of useful built-in components which allow to solve a wide range of tasks
+ without coding.
But you can need to extend provided functionality and develop your own components.
Let's look how to do it and when it can be useful for all kind of components step by step.
</p>
diff --git a/first-example.html b/first-example.html
index 86ed542..00fbd89 100644
--- a/first-example.html
+++ b/first-example.html
@@ -22,23 +22,6 @@
-->
<div class="col-md-8 second-column">
- <section id="tldr">
- <h2 class="section-title">TL;DR; <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Watch this quick video (10>03) for NLPCraft "Hello World!" example:
- </p>
- <div>
- <iframe
- width="514"
- height="289"
- src="https://www.youtube.com/embed/R7SRpxBxHHY?modestbranding=1"
- title="NLPCraft - Hello World!"
- frameborder="0"
- allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
- allowfullscreen>
- </iframe>
- </div>
- </section>
<section id="setup">
<h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
@@ -51,423 +34,255 @@
</p>
</section>
<section id="new_project">
- <h2 class="section-title">New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- We'll use <a href="/tools/script.html">NLPCraft CLI</a> to perform all management operations.
- Run <code>bin/nlpcraft.sh</code> script to create a new project stub:
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
</p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=LightSwitch --outputDir=~ --lang=scala --pkgName=demo
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/first_example_fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/LightSwitch</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- <div class="bq info">
- <p>
- <i class="fas fa-lightbulb"></i> Learn more about <a href="/tools/script.html">NLPCraft CLI</a>.
- </p>
- </div>
- </section>
- <section id="data_model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In NLPCraft a data model is an implementation of <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModel.html">NCModel</a>
- interface. You can move most of the model configuration out to an external JSON or YAML file and use
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelFileAdapter.html">NCModelFileAdapter</a> to
- load this external configuration when creating a new model.
- </p>
- <p>
- Open <code>src/main/resources/<b>light_switch.yaml</b></code> file and replace its content with
- the following model declaration:
- </p>
- <pre class="brush: js, highlight: [6, 22, 23, 26, 33, 41, 52]">
-id: "nlpcraft.lightswitch.ex"
-name: "Light Switch Example Model"
-version: "1.0"
-description: "NLI-powered light switch example model."
-
-macros:
- - name: "<ACTION>"
- macro: "{turn|switch|dial|let|set|get|put}"
- - name: "<KILL>"
- macro: "{shut|kill|stop|eliminate}"
- - name: "<ENTIRE_OPT>"
- macro: "{entire|full|whole|total|_}"
- - name: "<FLOOR_OPT>"
- macro: "{upstairs|downstairs|{1st|2nd|3rd|4th|5th|top|ground} floor|_}"
- - name: "<TYPE>"
- macro: "{room|closet|attic|loft|{store|storage} {room|_}}"
- - name: "<LIGHT>"
- macro: "{all|_} {it|them|light|illumination|lamp|lamplight}"
-
-enabledBuiltInTokens: []
-
-permutateSynonyms: true
-sparse: true
-
-elements:
- - id: "ls:loc"
- description: "Location of lights."
- synonyms:
- - "<ENTIRE_OPT> <FLOOR_OPT> {kitchen|library|closet|garage|office|playroom|{dinning|laundry|play} <TYPE>}"
- - "<ENTIRE_OPT> <FLOOR_OPT> {master|kid|children|child|guest|_} {bedroom|bathroom|washroom|storage} {<TYPE>|_}"
- - "<ENTIRE_OPT> {house|home|building|{1st|first} floor|{2nd|second} floor}"
-
- - id: "ls:on"
- groups:
- - "act"
- description: "Light switch ON action."
- synonyms:
- - "<ACTION> {on|up|_} <LIGHT> {on|up|_}"
- - "<LIGHT> {on|up}"
-
- - id: "ls:off"
- groups:
- - "act"
- description: "Light switch OFF action."
- synonyms:
- - "<ACTION> <LIGHT> {off|out}"
- - "{<ACTION>|<KILL>} {off|out} <LIGHT>"
- - "<KILL> <LIGHT>"
- - "<LIGHT> <KILL>"
- - "no <LIGHT>"
-
-intents:
- - "intent=ls term(act)={has(tok_groups, 'act')} term(loc)={# == 'ls:loc'}*"
+ <pre class="brush: js, highlight: []">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft LightSwitch Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
</pre>
- <p>There are number of important points here:</p>
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>lightswitch_model.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>LightSwitchModel.scala</code> - Scala class, model implementation.</li>
+ <li><code>LightSwitchModelSpec.scala</code> - Scala tests class, which allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 10, 14]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | lightswitch_model.yaml
+ | \--scala
+ | \--demo
+ | LightSwitchModel.scala
+ \--test
+ \--scala
+ \--demo
+ LightSwitchModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ We are going to start with declaring the static part of our model using YAML which we will later load using
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>light_switch.yaml</b></code>
+ file and replace its content with the following YAML:
+ </p>
+ <pre class="brush: js, highlight: [1, 10, 17, 25]">
+ macros:
+ "<ACTION>" : "{turn|switch|dial|let|set|get|put}"
+ "<KILL>" : "{shut|kill|stop|eliminate}"
+ "<ENTIRE_OPT>" : "{entire|full|whole|total|_}"
+ "<FLOOR_OPT>" : "{upstairs|downstairs|{1st|first|2nd|second|3rd|third|4th|fourth|5th|fifth|top|ground} floor|_}"
+ "<TYPE>" : "{room|closet|attic|loft|{store|storage} {room|_}}"
+ "<LIGHT>" : "{all|_} {it|them|light|illumination|lamp|lamplight}"
+
+ elements:
+ - id: "ls:loc"
+ description: "Location of lights."
+ synonyms:
+ - "<ENTIRE_OPT> <FLOOR_OPT> {kitchen|library|closet|garage|office|playroom|{dinning|laundry|play} <TYPE>}"
+ - "<ENTIRE_OPT> <FLOOR_OPT> {master|kid|children|child|guest|_} {bedroom|bathroom|washroom|storage} {<TYPE>|_}"
+ - "<ENTIRE_OPT> {house|home|building|{1st|first} floor|{2nd|second} floor}"
+
+ - id: "ls:on"
+ groups:
+ - "act"
+ description: "Light switch ON action."
+ synonyms:
+ - "<ACTION> {on|up|_} <LIGHT> {on|up|_}"
+ - "<LIGHT> {on|up}"
+
+ - id: "ls:off"
+ groups:
+ - "act"
+ description: "Light switch OFF action."
+ synonyms:
+ - "<ACTION> <LIGHT> {off|out|down}"
+ - "{<ACTION>|<KILL>} {off|out|down} <LIGHT>"
+ - "<KILL> <LIGHT>"
+ - "<LIGHT> <KILL>"
+ - "{out|no|off|down} <LIGHT>"
+ - "<LIGHT> {out|off|down}"
+ </pre>
+
<ul>
<li>
- <code>Line 6</code> defines several macros that are used later on throughout the model's elements
+ <code>Line 1</code> defines several macros that are used later on throughout the model's elements
to shorten the synonym declarations. Note how macros coupled with option groups
shorten overall synonym declarations 1000:1 vs. manually listing all possible word permutations.
</li>
<li>
- <code>Lines 22, 23</code> define model properties that allow for multi-word synonyms in this model
- to be
- <a class="not-code" target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelView.html#isSparse()">sparse</a> and
- <a class="not-code" target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelView.html#isPermutateSynonyms()">permutate</a> them for better detection. These two properties generally enable a free-form
- natural language comprehension.
- </li>
- <li>
- <code>Lines 26, 33, 41</code> define three model elements: the location of the light, and actions to turn
+ <code>Lines 10, 17, 25</code> define three model elements: the location of the light, and actions to turn
the light on and off. Action elements belong to the same group <code>act</code> which
- will be used in our intent (<code>line 42</code>). Note that these model elements are defined mostly
- through macros we have provided above.
- </li>
- <li>
- On <code>line 52</code> we define a non-conversational intent <code>ls</code> that requires
- one action (a token belonging to the group <code>act</code>) and optional list of light locations
- (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ will be used in our intent, defined in <code>LightSwitchModel</code> class. Note that these model
+ elements are defined mostly through macros we have defined above.
</li>
</ul>
- <p>
- Next, let's go ahead and add model's logic which we'll write in Scala with Java and Kotlin versions provided for reference.
- Open <code>src/main/scala/demo/<b>LightSwitch.scala</b></code> file and replace its content with the following
- code:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-scala" role="tab">Scala <img src="/images/scala-logo-h16.png" alt=""></a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-java" role="tab">Java <img src="/images/java2-h20.png" alt=""></a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-kotlin" role="tab">Kotlin <img src="/images/kotlin-h18.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-scala" role="tabpanel">
- <pre class="brush: scala, highlight: [5, 6, 7, 22, 23]">
-package demo
-
-import org.apache.nlpcraft.model.{NCIntentTerm, _}
-
-class LightSwitchModel extends NCModelFileAdapter("light_switch.yaml") {
- @NCIntentRef("ls")
- @NCIntentSample(Array(
- "Turn the lights off in the entire house.",
- "Switch on the illumination in the master bedroom closet.",
- "Get the lights on.",
- "Please, put the light out in the upstairs bedroom.",
- "Set the lights on in the entire house.",
- "Turn the lights off in the guest bedroom.",
- "Could you please switch off all the lights?",
- "Dial off illumination on the 2nd floor.",
- "Please, no lights!",
- "Kill off all the lights now!",
- "No lights in the bedroom, please.",
- "Kill the illumination now!"
- ))
- def onMatch(
- @NCIntentTerm("act") actTok: NCToken,
- @NCIntentTerm("loc") locToks: List[NCToken]
- ): NCResult = {
- val status = if (actTok.getId == "ls:on") "on" else "off"
- val locations =
- if (locToks.isEmpty)
- "entire house"
- else
- locToks.map(_.getOriginalText()).mkString(", ")
-
- // Add HomeKit, Arduino or other integration here.
-
- // By default - return a descriptive action string.
- NCResult.text(s"Lights '$status' in '${locations.toLowerCase}'.")
- }
-}
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-java" role="tabpanel">
- <pre class="brush: java">
-package demo;
-
-import org.apache.nlpcraft.model.*;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class LightSwitchJavaModel extends NCModelFileAdapter {
- public LightSwitchJavaModel() {
- // Loading the model from the file in the classpath.
- super("lightswitch_model.yaml");
- }
-
- @NCIntentRef("ls")
- @NCIntentSample({
- "Turn the lights off in the entire house.",
- "Switch on the illumination in the master bedroom closet.",
- "Get the lights on.",
- "Lights up in the kitchen.",
- "Please, put the light out in the upstairs bedroom.",
- "Set the lights on in the entire house.",
- "Turn the lights off in the guest bedroom.",
- "Could you please switch off all the lights?",
- "Dial off illumination on the 2nd floor.",
- "Please, no lights!",
- "Kill off all the lights now!",
- "No lights in the bedroom, please.",
- "Light up the garage, please!",
- "Kill the illumination now!"
- })
- NCResult onMatch(
- @NCIntentTerm("act") NCToken actTok,
- @NCIntentTerm("loc") List<NCToken> locToks) {
- String status = actTok.getId().equals("ls:on") ? "on" : "off";
- String locations = locToks.isEmpty() ?
- "entire house" :
- locToks.stream().map(t -> (String)t.meta("nlpcraft:nlp:origtext"))
- .collect(Collectors.joining(", "));
-
- // Add HomeKit, Arduino or other integration here.
-
- // By default - return a descriptive action string.
- return NCResult.text("Lights are [" + status + "] in [" + locations.toLowerCase() + "].");
- }
-}
-
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-kotlin" role="tabpanel">
- <pre class="brush: scala">
-package demo
-
-import org.apache.nlpcraft.model.*
-import java.util.stream.Collectors
-
-class LightSwitchKotlinModel : NCModelFileAdapter("lightswitch_model.yaml") {
- @NCIntentRef("ls")
- @NCIntentSample(
- "Turn the lights off in the entire house.",
- "Switch on the illumination in the master bedroom closet.",
- "Get the lights on.",
- "Lights up in the kitchen.",
- "Please, put the light out in the upstairs bedroom.",
- "Set the lights on in the entire house.",
- "Turn the lights off in the guest bedroom.",
- "Could you please switch off all the lights?",
- "Dial off illumination on the 2nd floor.",
- "Please, no lights!",
- "Kill off all the lights now!",
- "No lights in the bedroom, please.",
- "Light up the garage, please!",
- "Kill the illumination now!"
- )
- fun onMatch(
- @NCIntentTerm("act") actTok: NCToken,
- @NCIntentTerm("loc") locToks: List<NCToken>
- ): NCResult {
- val status = if (actTok.id == "ls:on") "on" else "off"
- val locations = if (locToks.isEmpty()) "entire house" else locToks.stream()
- .map { t: NCToken -> t.meta("nlpcraft:nlp:origtext") as String }
- .collect(Collectors.joining(", "))
-
- // Add HomeKit, Arduino or other integration here.
-
- // By default - return a descriptive action string.
- return NCResult.text("Lights are [" + status + "] in [" + locations.toLowerCase() + "].")
- }
-}
- </pre>
- </div>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>LightSwitchModel</code> class as well.
+ </p>
</div>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- <b>NOTES:</b>
+ Open <code>src/main/scala/demo/<b>LightSwitchModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [6, 7, 8, 10, 11, 14, 15, 22]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
+
+ class LightSwitchModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.lightswitch.java.ex", "LightSwitch Example Model", "1.0"),
+ new NCPipelineBuilder().withSemantic("en", "lightswitch_model.yaml").build
+ ):
+ @NCIntent("intent=ls term(act)={has(ent_groups, 'act')} term(loc)={# == 'ls:loc'}*")
+ def onMatch(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("act") actEnt: NCEntity,
+ @NCIntentTerm("loc") locEnts: List[NCEntity]
+ ): NCResult =
+ val status = if actEnt.getId == "ls:on" then "on" else "off"
+ val locations = if locEnts.isEmpty then "entire house" else locEnts.map(_.mkText).mkString(", ")
+
+ // Add HomeKit, Arduino or other integration here.=
+ // By default - just return a descriptive action string.
+ NCResult(s"Lights are [$status] in [${locations.toLowerCase}].")
+ </pre>
+ <p>
+ The intent callback logic is very simple - we return a descriptive confirmation message
+ back (explaining what lights were changed). With action and location detected, you can add
+ the actual light switching using HomeKit or Arduino devices. Let's review this implementation step by step:
</p>
<ul>
<li>
- On <code>line 5</code> we use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelFileAdapter.html">NCModelFileAdapter</a>
- to load its configuration from the external <code>light_switch.yaml</code> YAML file.
+ On <code>line 5</code> our class extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
</li>
<li>
- Our data model uses intent-based matching of the user input. Intent is a template and a callback for when
- a template is matched against user input:
- <ul>
- <li>
- <code>Line 6</code> uses <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentRef.html">@NCIntentRef</a>
- annotation to reference the intent defined in our model (<code>light_switch.yaml</code>, <code>line 42</code>).
- The intent matches
- two parts: first should be any token belonging to <code>act</code> group (i.e. "on" or "off" action),
- and the second should be a zero or more tokens with ID equal to <code>ls:loc</code> (i.e. optional list of locations).
- </li>
- <li>
- <code>Line 7</code> uses <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation that is used for <a href="#test">auto-testing</a> of the model.
- </li>
- </ul>
+ On <code>line 6</code> created model configuration with most default parameters.
</li>
<li>
- <code>Lines 22 and 23</code> use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntent.html">@NCIntentTerm</a>
- annotations to connect the matched result with a callback method formal parameters.
+ On <code>line 7</code> created pipeline, based on semantic model definition,
+ described in <code>lightswitch_model.yaml</code> file.
</li>
<li>
- When our intent is selected as a matching winner the method <code>onMatch(...)</code> on <code>line 20</code> is invoked. It
- returns a text response that indicates the light status at a requested location. That's where you can
- also add <a target=_ href="https://developer.apple.com/homekit/">HomeKit</a>,
- <a href="https://www.arduino.cc/" target=_>Ardunio</a> or other integrations to finish this implementation.
+ <code>Lines 10 and 11</code> annotates intents <code>ls</code> and its callback method <code>onMatch()</code>.
+ Intent <code>ls</code> requires one action (a token belonging to the group <code>act</code>) and optional list of light locations
+ (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ </li>
+ <li>
+ <code>Lines 14 and 15</code> map terms from detected intent to the formal method parameters of the
+ <code>onMatch()</code> method.
+ </li>
+ <li>
+ On the <code>line 22</code> the intent callback simply returns a confirmation message.
</li>
</ul>
- <div class="bq info">
- <p>
- <i class="fas fa-lightbulb"></i> Learn more about <a href="/data-model.html">Data Model</a> and <a href="/intent-matching.html">Intents</a>.
- </p>
- </div>
</section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <section id="build">
+ <h2 class="section-title">Build Project<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
<p>
- Once the model ready, run the Maven build:
+ Once the model ready, run the SBT build from the project folder:
</p>
- <pre class="brush: bash">
- $ cd ~/LightSwitch
- $ mvn clean package
+
+ <pre class="brush: scala, highlight: []">
+ $ sbt clean compile
</pre>
</section>
- <section id="server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Start local REST server, if it hasn't been started already:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/first_example_fig2.png">
- </p>
- </div>
- </div>
- <div class="bq info">
- <p>
- <i class="fas fa-lightbulb"></i> Learn more about <a href="/server-and-probe.html">Server <span class="amp">&</span> Data Probes</a>.
- </p>
- </div>
- </section>
- <section id="test">
+
+ <section id="testing">
<h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- Recall the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our code next to intent reference.
+ The test defined in <code>LightSwitchModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intent <code>ls</code>:
</p>
+ <pre class="brush: scala, highlight: [9, 11]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class LightSwitchModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new LightSwitchModel())) { client =>
+ def check(txt: String): Unit =
+ require(client.debugAsk(txt, "userId", true).getIntentId == "ls")
+
+ check("Turn the lights off in the entire house.")
+ check("Turn off all lights now")
+ check("Switch on the illumination in the master bedroom closet.")
+ check("Get the lights on.")
+ check("Off the lights on the 1st floor")
+ check("Lights up in the kitchen.")
+ check("Please, put the light out in the upstairs bedroom.")
+ check("Set the lights on in the entire house.")
+ check("Turn the lights off in the guest bedroom.")
+ check("Could you please switch off all the lights?")
+ check("Dial off illumination on the 2nd floor.")
+ check("Turn down lights in 1st floor bedroom")
+ check("Lights on at second floor kitchen")
+ check("Please, no lights!")
+ check("Kill off all the lights now!")
+ check("Down the lights in the garage")
+ check("Lights down in the kitchen!")
+ check("Turn up the illumination in garage and master bedroom")
+ check("Turn down all the light now!")
+ check("No lights in the bedroom, please.")
+ check("Light up the garage, please!")
+ check("Kill the illumination now!")
+ }
+ }
+ </pre>
+ <ul>
+ <li>
+ <code>Line 9</code> creates the client for our model.
+ </li>
+ <li>
+ <code>Line 11</code> calls a special method <code>debugAsk()</code>.
+ It allows to check the winning intent and its callback parameters without actually
+ calling the intent.
+ </li>
+ <li>
+ <code>Lines 13-34</code> define all the test input sentences that should all
+ trigger <code>ls</code> intent.
+ </li>
+ </ul>
<p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
</p>
- <p>
- Run model auto-validator:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-test-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-test-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-test-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/LightSwitch/target/classes --mdls=demo.LightSwitchModel
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-test-out" role="tabpanel">
- <p></p>
- <p>
- <img style="max-width: 910px !important;" class="img-fluid" alt="" src="/images/first_example_fig4.png">
- </p>
- </div>
- </div>
+ <pre class="brush: scala, highlight: []">
+ $ sbt executeTests
+ </pre>
</section>
<section>
- <h2 class="section-title">Congratulation! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- You've created your first data model, started the
- REST server and tested the model using built-in model auto-validator.
+ You've created light switch data model and tested it.
</p>
</section>
</div>
@@ -475,10 +290,10 @@
<ul class="side-nav">
<li class="side-nav-title">On This Page</li>
<li><a href="#new_project">New Project</a></li>
- <li><a href="#data_model">Data Model</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#server">Start Server</a></li>
- <li><a href="#test">Testing</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#build">Build Project</a></li>
+ <li><a href="#testing">Testing</a></li>
{% include quick-links.html %}
</ul>
</div>
diff --git a/semantic.html b/semantic.html
deleted file mode 100644
index daad390..0000000
--- a/semantic.html
+++ /dev/null
@@ -1,672 +0,0 @@
----
-active_crumb: Docs
-layout: documentation
-id: semantic
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Semantic parser<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- Semantic entity parser
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
- is the implementation of <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>,
- which in turn is component of the model pipeline <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>.
- In spite of that it is just one of defined in NLPCraft built-in component, it deserves a special mention.
- This parser provides simple but very powerful way to find domain specific data in the input text.
- It defines list of <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a>
- which are represent <a href="https://en.wikipedia.org/wiki/Named-entity_recognition">Named entities</a>.
- We will name this list as <code>Semantic Model</code>.
-
- Let's talk a little bit more about <a href="https://en.wikipedia.org/wiki/Named-entity_recognition">Named entities</a>.
- </p>
- </section>
-
- <section id="ne">
- <h2 class="section-title">Named Entities<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Named entity, also known as a semantic element or a token, is one of the main a components defined by the NLPCraft data model.
- A named entity is one or more individual words that have a consistent semantic meaning and typically denote a
- real-world object, such as persons, locations, number, date and time, organizations, products, etc. Such
- object can be abstract or have a physical existence.
- </p>
- <p>
- For example, in the following sentence: TODO: PIC
- </p>
- <figure>
- <img alt="named entities" class="img-fluid" src="/images/named-entities.png">
- <figcaption><b>Fig 2.</b> Named Entities</figcaption>
- </figure>
- <p>
- the following named entities can be detected:
- </p>
- <table class="gradient-table">
- <thead>
- <tr>
- <th>Words</th>
- <th>Type</th>
- <th>Normalized Value</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><b>Top 20</b></td>
- <td><code>user:element:1</code></td>
- <td>top 20</td>
- </tr>
- <tr>
- <td><b>best pages</b></td>
- <td><code>user:element:2</code></td>
- <td>best pages</td>
- </tr>
- <tr>
- <td><b>California USA</b></td>
- <td><code>stanford:city</code></td>
- <td>USA, California</td>
- </tr>
- <tr>
- <td><b>last 3 months</b></td>
- <td><code>stanford:date</code></td>
- <td>1/1/2021 - 4/1/2021</td>
- </tr>
- </tbody>
- </table>
- <p>
- In most cases named entities will have associated <em>normalized value</em>. It is especially important for named entities that have many
- notational forms such as time and date, currency, geographical locations, etc. For example, <code>New York</code>,
- <code>New York City</code> and <code>NYC</code> all refer to the same "New York City, NY USA" location which is a standard normalized form.
- </p>
- <p>
- The process of detecting named entities is called Named Entity Recognition (NER). There are many ways of how a certain named entity can be detected: through list of synonyms, by name, rule-based or by using
- statistical techniques like neural networks with large corpus of predefined data. NLPCraft natively supports synonym-based
- named entities definition as well as the ability to compose new named entities through powerful <a href="/intent-matching.html">Intent Definition Language</a> (IDL)
- combining other named entities including named entities from
- such OpenNLP, or Stanford CoreNLP, look at the <a href="built-in-components.html">Built component</a> chapter.
- </p>
- <p>
- Named entities allow you to abstract from basic linguistic forms like nouns and verbs to deal with the higher level semantic
- abstractions like geographical location or time when you are trying to understand the meaning of the sentence.
- One of the main goals of named entities is to act as an input ingredients for <a href="/intent-matching.html">intent matching</a>.
- </p>
- <div class="bq info">
- <p>
- <b>😀 User Input → Named Entities → Parsing Variants → Intent Matcher → Winning Intent 🚀</b>
- </p>
- <p>
- User input is parsed into the list of named entities. That list is then further transformed into one or more
- parsing variants where each variant represents a particular order and combination of detected named entities.
- Finally, the list of variants act as an input to intent matching where each variant is matched against every intent
- in the process of detecting the best matching intent for the original user input.
- </p>
- </div>
- </section>
-
- <section id="elements">
- <h2 class="section-title">Semantic elements<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> represents
- NER element for its detection un the user input.
- <p>
-
- <div class="bq info">
- <p>
- <b>Semantic Element <span class="amp">&</span> Named Entity <span class="amp">&</span> Token</b>
- </p>
- <p>
- Terms 'semantic element', 'named entity' and 'token' are used throughout this documentation relatively interchangeably:
- </p>
- <dl>
- <dt>Semantic Element</dt>
- <dd>
- Denotes a named entity <em>declared</em> in NLPCraft model.
- </dd>
- <dt>Token</dt>
- <dd>
- Denotes a semantic element that was <em>detected</em> by NLPCraft in the user input.
- </dd>
- <dt>Named Entity</dt>
- <dd>
- Denotes a classic term, i.e. one or more individual words that have a
- consistent semantic meaning and typically define a real-world object.
- </dd>
- </dl>
- </div>
-
- <p>
- Each <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a>
- is presented by <code>id</code>, <code>groups</code>, <code>synonyms</code>, <code>values</code> and <code>properties</code>.
- <p>
- <span id="synonyms" class="section-sub-title">Synonyms <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- NLPCraft uses fully deterministic named entity recognition and is not based on statistical approaches that
- would require pre-existing marked up data sets and extensive training. For each semantic element you can either provide a
- set of synonyms to match on or specify a piece of code that would be responsible for detecting that named
- entity (discussed below). A synonym can have one or more individual words. Note that element's ID is its
- implicit synonym so that even if no additional synonyms are defined at least one synonym always exists. Note
- also that synonym matching is performed on <em>normalized</em> and <em>stemmatized</em> forms of both
- a synonym and user input.
- </p>
-
- <p>
- Here's an example of a simple semantic element definition in JSON:
- </p>
- <pre class="brush: js, highlight: [6,7,8,9,10,11,12]">
- ...
- "elements": [
- {
- "id": "transport.vehicle",
- "description": "Transportation vehicle",
- "synonyms": [
- "car",
- "truck",
- "light duty truck"
- "heavy duty truck"
- "sedan",
- "coupe"
- ]
- }
- ]
- ...
- </pre>
- <p>
- While adding multi-word synonyms looks somewhat
- trivial - in real models, the naive approach can lead to thousands and even tens of thousands of
- possible synonyms due to words, grammar, and linguistic permutations - which quickly becomes untenable if
- performed manually.
- </p>
- <p>
- NLPCraft provides an effective tool for a compact synonyms representation. Instead of listing all possible
- multi-word synonyms one by one you can use combination of following techniques:
- </p>
- <ul>
- <li><a href="#macros">Macros</a></li>
- <li><a href="#regex">Regular expressions</a></li>
- <li><a href="#option-groups">Option Groups</a></li>
- </ul>
- <p>
- Each whitespace separated string in the synonym can be either a regular word (like in the above transportation example
- where it will be matched on using its normalized and stemmatized form) or one of the above expression.
- </p>
- <p>
- Note that this synonyms definition is also used in the following
- <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> methods:
- </p>
- <ul>
- <li><code>getSynonyms()</code> - gets synonyms to match on.</li>
- <li><code>getValues()</code> - get values to match on (see <a href="#values">below</a>).</li>
- </ul>
- <span id="values" class="section-sub-title">Element Values <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Semantic element can have an optional set of special synonyms called <em>values</em> or "proper nouns" for this element.
- Unlike basic synonyms, each value is a pair of a name and a set of standard synonyms by which that value,
- and ultimately its element, can be recognized in the user input. Note that the value name itself acts as an
- implicit synonym even when no additional synonyms added for that value.
- </p>
- <p>
- When a semantic element is recognized it is made available to the model's matching logic as an instance of
- the <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a> interface.
- This interface has a method
- <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">getValue()</a> which
- returns the name of the value, if any, by which
- that semantic element was recognized. That value name can be further used in intent matching.
- </p>
- <p>
- To understand the importance of the values consider the following changes to our transportation
- example model:
- </p>
- <pre class="brush: js, highlight: [19,20,21,22,23,24,25,26,27,28,29,30]">
- ...
- "macros": [
- {
- "name": "<TRUCK_TYPE>",
- "macro": "{light duty|heavy duty|half ton|1/2 ton|3/4 ton|one ton|super duty}"
- }
- ]
- "elements": [
- {
- "id": "transport.vehicle",
- "description": "Transportation vehicle",
- "synonyms": [
- "car",
- "{<TRUCK_TYPE>|_} {pickup|_} truck"
- "sedan",
- "coupe"
- ],
- "values": [
- {
- "value": "mercedes",
- "synonyms": ["mercedes-ben{z|s}", "mb", "ben{z|s}"]
- },
- {
- "value": "bmw",
- "synonyms": ["{bimmer|bimer|beemer}", "bayerische motoren werke"]
- }
- {
- "value": "chevrolet",
- "synonyms": ["chevy"]
- }
- ]
- }
- ]
- ...
- </pre>
- <p>
- With that setup <code>transport.vehicle</code> element will be recognized by any of the following input string:
- </p>
- <ul>
- <li><code>car</code></li>
- <li><code>benz</code> (with value <code>mercedes</code>)</li>
- <li><code>3/4 ton pickup truck</code></li>
- <li><code>light duty truck</code></li>
- <li><code>chevy</code> (with value <code>chevrolet</code>)</li>
- <li><code>bimmer</code> (with value <code>bmw</code>)</li>
- <li><code>transport.vehicle</code></li>
- </ul>
- <span id="groups" class="section-sub-title">Element Groups <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Each semantic element always belongs to one or more groups. Semantic element provides its groups via
- <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">getGroups()</a> method.
- By default, if element group is not specified, the element ID will act as its default group ID.
- Group membership is a quick and easy way to organise similar semantic elements together and use this
- categorization in <a href="/intent-matching.html">IDL</a> intents.
- </p>
- <p>
- Note that the proper grouping of the elements is also necessary for the correct operation of
- Short-Term-Memory (STM) in the conversational context. Consider a
- <a href="/apis/latest/org/apache/nlpcraft/NCToken.html">NCToken</a> that
- represents a previously found semantic element that is stored in the conversation. Such token
- will be overridden in the conversation by the more <b>recent token</b>
- from the <b>same group</b> - a critical rule of maintaining the proper conversational context.
- See
- <a href="/apis/latest/org/apache/nlpcraft/NCConversation.html">NCConversation</a>
- for mode details.
- </p>
- </section>
-
- <section id="macros">
- <h2 class="section-title">Macros <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Listing all possible multi-word synonyms for a given element can be a time-consuming task. Macros
- together with option groups allow for significant simplification of this task.
- Macros allow you to give a name to an often used set of words or option groups and reuse it without
- repeating those words or option groups again and again. A model provides a list of macros via
- <a href="/apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">macros</a> method.
- Each macro has a name in a form of <code><X></code> where <code>X</code>
- is any string, and a string value. Note that macros can be nested (but not recursive), i.e. macro value can include
- references to other macros. When macro name <code>X</code> is encountered in the synonym it gets recursively
- replaced with its value.
- </p>
- <p>
- Here's a code snippet of macro definitions using JSON definition:
- </p>
- <pre class="brush: js">
- "macros": [
- {
- "name": "<A>",
- "macro": "aaa"
- },
- {
- "name": "<B>",
- "macro": "<A> bbb"
- },
- {
- "name": "<C>",
- "macro": "<A> bbb {z|w}"
- }
- ]
- </pre>
- <span id="option-groups" class="section-sub-title">Option Groups <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Option groups are similar to wildcard patterns that operates on a single word base. One line of
- option group expands into one or more individual synonyms. Option groups is the key mechanism for shortened
- synonyms notation. The following examples demonstrate how to use option groups.
- </p>
- <p>
- Consider the following macros defined below (note that macros <code><B></code> and <code><C></code>
- are nested):
- </p>
- <table class="gradient-table">
- <thead>
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code><A></code></td>
- <td><code>aaa</code></td>
- </tr>
- <tr>
- <td><code><B></code></td>
- <td><code><A> bbb</code></td>
- </tr>
- <tr>
- <td><code><C></code></td>
- <td><code><A> bbb {z|w}</code></td>
- </tr>
- </tbody>
- </table>
- <p>
- Then the following option group expansions will occur in these examples:
- </p>
- <table class="gradient-table">
- <thead>
- <tr>
- <th>Synonym</th>
- <th>Synonym Expansions</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code><A> {b|_} c</code></td>
- <td>
- <code>"aaa b c"</code><br>
- <code>"aaa c"</code>
- </td>
- </tr>
- <tr>
- <td><code><A> {b|a}[1,2] c</code></td>
- <td>
- <code>"aaa b c"</code><br>
- <code>"aaa b b c"</code><br>
- <code>"aaa a c"</code><br>
- <code>"aaa a a c"</code><br>
- <code>"aaa c"</code>
- </td>
- </tr>
- <tr>
- <td>
- <code><B> {b|_} c</code><br>
- or<br>
- <code><B> {b}[0,1] c</code>
- </td>
- <td>
- <code>"aaa bbb b c"</code><br>
- <code>"aaa bbb c"</code>
- </td>
- </tr>
- <tr>
- <td><code>{b|\{\_\}}</code></td>
- <td>
- <code>"b"</code><br>
- <code>"b {_}"</code>
- </td>
- </tr>
- <tr>
- <td><code>a {b|_}. c</code></td>
- <td>
- <code>"a b. c"</code><br>
- <code>"a . c"</code>
- </td>
- </tr>
- <tr>
- <td><code>a .{b, |_}. c</code></td>
- <td>
- <code>"a .b, . c"</code><br>
- <code>"a .. c"</code>
- </td>
- </tr>
- <tr>
- <td><code>
- {% raw %}a {{b|c}|_}.{% endraw %}</code></td>
- <td>
- <code>"a ."</code><br>
- <code>"a b."</code><br>
- <code>"a c."</code>
- </td>
- </tr>
- <tr>
- <td><code>a {% raw %}{{{<C>}}|{_}}{% endraw %} c</code></td>
- <td>
- <code>"a aaa bbb z c"</code><br>
- <code>"a aaa bbb w c"</code><br>
- <code>"a c"</code>
- </td>
- </tr>
- <tr>
- <td><code>{% raw %}{{{a}}} {b||_|{{_}}||_}{% endraw %}</code></td>
- <td>
- <code>"a b"</code><br>
- <code>"a"</code>
- </td>
- </tr>
- </tbody>
- </table>
- <p>
- Specifically:
- </p>
- <ul>
- <li><code>{A|B}</code> denotes either <code>A</code> or <code>B</code>.</li>
- <li>
- <code>{A|B|_}</code> denotes either <code>A</code> or <code>B</code> or nothing.
- <ul>
- <li>Symbol <code>_</code> cam appear anywhere in the list of options, i.e. <code>{A|B|_}</code> is equal to <code>{A|_|B}</code>.</li>
- </ul>
- </li>
- <li>
- <code>{C}[x,y]</code> denotes an option group with quantifier, i.e. group <code>C</code> appearing from <code>x</code> to <code>y</code> times inclusive.
- <ul>
- <li>For example, <code>{C}[1,3]</code> is the same as <code>{C|C C|C C C}</code> notation.</li>
- <li>Note that <code>{C|_}</code> is equal to <code>{C}[0,1]</code></li>
- </ul>
- </li>
- <li>Excessive curly brackets are ignored, when safe to do so.</li>
- <li>Macros cannot be recursive but can be nested.</li>
- <li>Option groups can be nested.</li>
- <li>
- <code>'\'</code> (backslash) can be used to escape <code>'{'</code>, <code>'}'</code>, <code>'|'</code> and
- <code>'_'</code> special symbols used by the option groups.
- </li>
- <li>Excessive whitespaces are trimmed when expanding option groups.</li>
- </ul>
- <p>
- We can rewrite our transportation semantic element in a more efficient way using macros and option groups.
- Even though the actual length of definition hasn't changed much it now auto-generates many dozens of synonyms
- we would have to write out manually otherwise:
- </p>
- <pre class="brush: js, highlight: [4,5,14]">
- ...
- "macros": [
- {
- "name": "<TRUCK_TYPE>",
- "macro": "{ {light|super|heavy|medium} duty|half ton|1/2 ton|3/4 ton|one ton}"
- }
- ]
- "elements": [
- {
- "id": "transport.vehicle",
- "description": "Transportation vehicle",
- "synonyms": [
- "car",
- "{<TRUCK_TYPE>|_} {pickup|_} truck"
- "sedan",
- "coupe"
- ]
- }
- ]
- ...
- </pre>
- <span id="regex" class="section-sub-title">Regular Expressions <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Any individual synonym word that starts and ends with <code>//</code> (two forward slashes) is
- considered to be Java regular expression as defined in <code>java.util.regex.Pattern</code>. Note that
- regular expression can only span a single word, i.e. only individual words from the user input will be
- matched against given regular expression and no whitespaces are allowed within regular expression. Note
- also that option group special symbols <code>{</code>, <code>}</code>,
- <code>|</code> and <code>_</code> have to be escaped in the regular expression using <code>\</code>
- (backslash).
- </p>
- <p>
- For example, the following synonym:
- </p>
- <pre class="brush: js">
- "synonyms": [
- "{foo|//[bar].+//}}"
- ]
- </pre>
- <p>
- will match word <code>foo</code> or any other strings that start with <code>bar</code> as long as
- this string doesn't contain whitespaces.
- </p>
- <div class="bq info">
- <b>Regular Expressions Performance</b>
- <p>
- It's important to note that regular expressions can significantly affect the performance of the
- NLPCraft processing if used uncontrolled. Use it with caution and test the performance
- of your model to ensure it meets your requirements.
- </p>
- </div>
- </section>
-
- <section id="usage">
- <h2 class="section-title">Usage examples<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
-
- <p>
- The following examples shows how to build model programmatically.
- </p>
-
- <pre class="brush: scala, highlight: [3, 5, 10]">
- val mdl = new NCModelAdapter(
- NCModelConfig("test.id", "Test Model", "1.0"),
- new NCPipelineBuilder().withSemantic(
- "en",
- Map(
- "<OF>" -> "{of|for|per}",
- "<CUR>" -> "current|present|now|local}",
- "<TIME>" -> "{time <OF> day|day time|date|time|moment|datetime|hour|o'clock|clock|date time|date and time|time and date}",
- )
- List(
- new NCSemanticElement():
- override def getId: String = "time"
- override def getSynonyms: Set[String] = Set("{<CUR>|_} <TIME>", "what <TIME> {is it now|now|is it|_}" )
- )
- ).build
- )
- </pre>
- <ul>
- <li>
- <code>Line 5</code> shows <code>macro</code> parameter usage.
- </li>
- <li>
- <code>Line 10</code> shows <code>macro</code> list of <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticElement.html">NCSemanticElement</a> parameter usage.
- </li>
- <li>
- Note that usage <code>withSemantic()</code> method which represented on <code>line 3</code> i optional.
- You can add <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCNLPEntityParser.html">NCNLPEntityParser</a>
- as usual <a href="apis/latest/org/apache/nlpcraft/NCEntityParser.html">NCEntityParser</a>
- when you define your <a href="apis/latest/org/apache/nlpcraft/NCPipeline.html">NCPipeline</a>.
- </li>
- </ul>
-
- <p>
- The following examples is based on YAML semantic elements representation.
- </p>
-
- <pre class="brush: js, highlight: []">
- macros:
- "<OF>": "{of|for|per}"
- "<CUR>": "{current|present|now|local}"
- "<TIME>": "{time <OF> day|day time|date|time|moment|datetime|hour|o'clock|clock|date time|date and time|time and date}"
- elements:
- - id: "x:time"
- description: "Date and/or time token indicator."
- synonyms:
- - "{<CUR>|_} <TIME>"
- - "what <TIME> {is it now|now|is it|_}"
- </pre>
-
- <ul>
- <li>
- Same macros and the same element as in previous example are defined here in
- <code>time_model.yaml</code> YAML file.
- </li>
- </ul>
- <pre class="brush: scala, highlight: [3]">
- val mdl = new NCModelAdapter(
- NCModelConfig("test.id", "Test Model", "1.0"),
- new NCPipelineBuilder().withSemantic("en", "time_model.yaml").build
- )
- </pre>
- <ul>
- <li>
- <code>Line 3</code> makes semantic model which defined in <code>time_model.yaml</code> YAML file.
- </li>
- </ul>
-
- <p>
- If you want to use <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
- with not English language, you have to provide custom
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a> and
- <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
- implementations for required language. Look at the <a href="examples/light_switch_fr.html">Light Switch FR</a>
- for more details.
- </p>
-
- <pre class="brush: scala, highlight: [4, 7, 8]">
- package demo
-
- import opennlp.tools.stemmer.snowball.SnowballStemmer
- import demo.nlp.token.parser.NCFrTokenParser
- import org.apache.nlpcraft.nlp.parsers.*
-
- class NCFrSemanticEntityParser(src: String) extends NCSemanticEntityParser(
- new NCSemanticStemmer:
- private val stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.FRENCH)
- override def stem(txt: String): String = stemmer.synchronized { stemmer.stem(txt.toLowerCase).toString }
- ,
- new NCFrTokenParser(),
- mdlSrcOpt = Option(src)
- )
- </pre>
- <ul>
- <li>
- <code>Line 4</code> includes <code>NCFrTokenParser</code> import.
- Its custom <a href="apis/latest/org/apache/nlpcraft/NCTokenParser.html">NCTokenParser</a>
- implementation for French language, described here: <a href="examples/light_switch_fr.html">Light Switch FR</a>.
- </li>
- <li>
- <code>Line 8</code> defines custom <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticStemmer.html">NCSemanticStemmer</a>
- implementation for French language.
- </li>
- <li>
- As you can see, <code>NCFrSemanticEntityParser</code> is very simple extension of
- <a href="apis/latest/org/apache/nlpcraft/nlp/parsers/NCSemanticEntityParser.html">NCSemanticEntityParser</a>
- base class, look at <code>line 7</code>.
- </li>
- </ul>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#ne">Named Entities</a></li>
- <li><a href="#elements">Semantic elements</a></li>
- <li><a href="#macros">Macros</a></li>
- <li><a href="#usage">Usage examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
