Merge branch 'master' into NLPCRAFT-513
diff --git a/_config.yml b/_config.yml
index 4e2da93..16ddc88 100644
--- a/_config.yml
+++ b/_config.yml
@@ -27,8 +27,9 @@
# Manually update to the latest version:
# ===================
-latest_version: 0.9.0
-java_client_latest_version: 0.9.0
+latest_version: 1.0.0
+# TODO: remove it.
+java_client_latest_version: 1.0.0
# ===================
# SASS settings.
diff --git a/_includes/left-side-menu.html b/_includes/left-side-menu.html
index c62d1eb..64e44c5 100644
--- a/_includes/left-side-menu.html
+++ b/_includes/left-side-menu.html
@@ -60,75 +60,25 @@
{% endif %}
</li>
<li>
- {% if page.id == "server_and_probe" %}
- <a class="active" href="/server-and-probe.html">Server <span class="amp">&</span> Probe</a>
- {% else %}
- <a href="/server-and-probe.html">Server <span class="amp">&</span> Probe</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "metrics" %}
- <a class="active" href="/metrics-and-tracing.html">Metrics <span class="amp">&</span> Tracing</a>
- {% else %}
- <a href="/metrics-and-tracing.html">Metrics <span class="amp">&</span> Tracing</a>
- {% endif %}
- </li>
- <li>
{% if page.id == "integrations" %}
<a class="active" href="/integrations.html">Integrations</a>
{% else %}
<a href="/integrations.html">Integrations</a>
{% endif %}
</li>
- <li>
- {% if page.id == "rest" %}
- <a class="active" href="/using-rest.html">REST API</a>
- {% else %}
- <a href="/using-rest.html">REST API</a>
- {% endif %}
- </li>
- <li class="side-nav-title">Tools</li>
- <li>
- {% if page.id == "script" %}
- <a class="active" href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a>
- {% else %}
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "test_framework" %}
- <a class="active" href="/tools/test_framework.html">Test Framework</a>
- {% else %}
- <a href="/tools/test_framework.html">Test Framework</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "embedded_probe" %}
- <a class="active" href="/tools/embedded_probe.html">Embedded Probe</a>
- {% else %}
- <a href="/tools/embedded_probe.html">Embedded Probe</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "sql_model_gen" %}
- <a class="active" href="/tools/sql_model_gen.html">SQL Model Generator</a>
- {% else %}
- <a href="/tools/sql_model_gen.html">SQL Model Generator</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "syn_tool" %}
- <a class="active" href="/tools/syn_tool.html">Synonyms Tool</a>
- {% else %}
- <a href="/tools/syn_tool.html">Synonyms Tool</a>
- {% endif %}
- </li>
<li class="side-nav-title">Examples</li>
<li>
- {% if page.id == "alarm_clock" %}
- <a class="active" href="/examples/alarm_clock.html">Alarm Clock</a>
+ {% if page.id == "calculator" %}
+ <a class="active" href="/examples/calculator.html">Calculator</a>
{% else %}
- <a href="/examples/alarm_clock.html">Alarm Clock</a>
+ <a href="/examples/calculator.html">Calculator</a>
+ {% endif %}
+ </li>
+ <li>
+ {% if page.id == "time" %}
+ <a class="active" href="/examples/time.html">Time</a>
+ {% else %}
+ <a href="/examples/time.html">Time</a>
{% endif %}
</li>
<li>
@@ -139,17 +89,24 @@
{% endif %}
</li>
<li>
- {% if page.id == "weather_bot" %}
- <a class="active" href="/examples/weather_bot.html">Weather Bot</a>
+ {% if page.id == "light_switch_fr" %}
+ <a class="active" href="/examples/light_switch_fr.html">Light Switch FR</a>
{% else %}
- <a href="/examples/weather_bot.html">Weather Bot</a>
+ <a href="/examples/light_switch_fr.html">Light Switch FR</a>
{% endif %}
</li>
<li>
- {% if page.id == "sql_model" %}
- <a class="active" href="/examples/sql_model.html">SQL Model</a>
+ {% if page.id == "light_switch_ru" %}
+ <a class="active" href="/examples/light_switch_ru.html">Light Switch RU</a>
{% else %}
- <a href="/examples/sql_model.html">SQL Model</a>
+ <a href="/examples/light_switch_ru.html">Light Switch RU</a>
+ {% endif %}
+ </li>
+ <li>
+ {% if page.id == "pizzeria" %}
+ <a class="active" href="/examples/pizzeria.html">Pizzeria</a>
+ {% else %}
+ <a href="/examples/pizzeria.html">Pizzeria</a>
{% endif %}
</li>
</ul>
diff --git a/docs.html b/docs.html
index 78e2e38..fb19d6a 100644
--- a/docs.html
+++ b/docs.html
@@ -25,12 +25,10 @@
<section id="overview">
<h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- Apache NLPCraft is a JVM-based <a target=_blank href="https://www.apache.org/licenses/">open source</a> library
- for adding a natural language interface to modern applications. It enables people to interact with your products using voice or text. NLPCraft can connect with
- any private or public data source, and has no hardware or software lock-ins. Its design is based on advanced
- <a href="/intent-matching.html">Intent Definition Language</a> (IDL) for defining non-trivial intents and a fully deterministic intent matching
- algorithm for the input utterances. You can build intents for NLPCraft using any JVM-based languages like Java, Scala, Kotlin, Groovy, etc. NLPCraft
- exposes REST APIs for integration with end-user applications.
+ Apache NLPCraft is an <a target=_blank href="https://www.apache.org/licenses/">open source</a> Scala library for adding a natural language interface to modern applications.
+ It enables people to interact with your products using voice or text.
+ Its design is based on advanced <a href="/intent-matching.html">Intent Definition Language</a> (IDL) for defining non-trivial intents and
+ a fully deterministic intent matching algorithm for the input utterances.
</p>
<p>
One of the key features of NLPCraft is its use of <a href="/intent-matching.html">IDL</a> coupled with deterministic intent matching that are tailor made for
@@ -38,107 +36,123 @@
approach with time consuming corpora development and model training - resulting in much a
<em>simpler <span class="amp">&</span> faster</em> implementation.
</p>
+
<p>
- Another key aspect of NLPCraft is its initial focus on processing English language. Although it may sound
- counterintuitive, this narrower initial focus enables NLPCraft to deliver unprecedented ease of use combined with
- unparalleled comprehension capabilities for English input out-of-the-box. It avoids academic, watered down functionality or overly
- complicated configuration and usage - following on project's <em>"built for engineers by engineers"</em> ethos.
- English language is spoken by more
- than a billion people on this planet and is de facto standard global language of the business and commerce.
+ NlpCraft library contains two base elements: <code>Model</code> and <code>Client</code>.
</p>
+
+ <ul>
+ <li>
+ <code>Model</code> is domain specific object which responsible for user input interpretation. Model contains intents, defined via NlpCraft IDL with related code callbacks. Intent is user defined callback and rule, according to which this callback should be called. Rule is most often some template, based on expected set of entities in user input, but it can be more flexible.
+ </li>
+
+ <li>
+ <code>Client</code> is object, which allows to communicate with given model. Main methods are user input processing and control of communication session.
+ </li>
+ </ul>
+
+ <p>Typical part of code:</p>
+
+ <pre class="brush: scala, highlight: []">
+ // Prepares domain model.
+ val mdl = new CustomNlpModel()
+
+ // Prepares client for given model.
+ val client = new NCModelClient(mdl)
+
+ // Sends text request to model by user ID "userId".
+ val result = client.ask("Some user command", "userId")
+
+ // Clears dialog session for user with ID "userId".
+ client.clearDialog("userId")
+ </pre>
+
<p>
- So, how does it work in a nutshell?
- </p>
- <p>
- When using NLPCraft you will be dealing with three main components:
+ Model definition includes two parts:
</p>
<ul>
- <li><a href="#data-model">Data model</a></li>
- <li><a href="#data-probe">Data probe</a></li>
- <li><a href="#server">REST Server</a></li>
+ <li>
+ <code>Configuration</code>. Static configuration parameters including name, version, etc.
+ </li>
+ <li>
+ <code>Pipeline</code>. Most important component, which defines user input processing chain.
+ <code>Pipeline</code> can be based on standard and custom user defined components.
+ </li>
</ul>
- <figure>
- <img class="img-fluid" src="/images/homepage-fig1.1.png" alt="">
- <figcaption><b>Fig 1.</b> NLPCraft Architecture</figcaption>
- </figure>
- </section>
- <section id="data-model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
<p>
- NLPCraft employs a <em>model-as-a-code</em> approach where everything you do in NLPCraft is part of your source code. Data model is simply an implementation of
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModel.html">NCModel</a> Java interface that
- can be developed using any JVM programming language like Java, Scala, Kotlin or Groovy.
- Data model defines named entities, various configuration properties as well as intents to interpret user input. Model-as-a-code natively supports
- any software lifecycle tools and frameworks in Java ecosystem.
+ Before looking at pipeline elements more throughly, let's start with terminology.
</p>
+
+ <ul>
+ <li>
+ <code>Token</code>. It is simple string, part of user input, which split according to some rules, for instance by spaces and some additional conditions, which depends on language and some expectations.
+ So user input "<b>Where is it?</b>" contains four tokens: "<b>Where</b>", "<b>is</b>", "<b>it</b>", "<b>?</b>".
+ </li>
+ <li>
+ <code>Entity</code>. According to wikipedia, named entity is a real-world object, such as a person, location, organization, product, etc., that can be denoted with a proper name. It can be abstract or have a physical existence. Each entity can contain one or more tokens.
+ </li>
+ <li>
+ <code>Variant</code>. List of entities. Potentially, each token can be recognized as different entities, so user input can be processed as set of variants. For example user input "Mercedes" can be processed as 2 variants, both of them contains single element list of entities: car brand or Spanish family name.
+ </li>
+ </ul>
+
<p>
- Declarative portion of the model can be stored in a separate JSON or YAML file
- for simpler maintenance. There are no practical limitation on how complex or simple a model
- can be, or what other tools it can use. Data models use <a href="/intent-matching.html">intents</a> to match the user input.
+ Back to pipeline. Pipeline should be created based in following components:
</p>
+ <ul>
+ <li>
+ <code>Token parser</code>. Mandatory NLP component, it is required for parsing plain text, user input, and split this text into tokens list. NlpCraft provides default EN implementation of token parser. Also, project contain various examples for FR and RU languages.
+ </li>
+ <li>
+ <code>Tokens enrichers</code> optional list. Tokens enricher is component which allows to add additional properties to prepared tokens, like part of speech, quote, stop-words flags or any other. NlpCraft provides default set of EN tokens enrichers implementations.
+ </li>
+ <li>
+ <code>Tokens validators</code> optional list. Tokens validator is user defined component, where tokens are inspected and exception can be thrown from user code to break user input processing.
+ </li>
+ <li>
+ <code>Entity parsers</code> mandatory list. At least one entity parser must be defined. Having prepared tokens as input, each entity parser tries to find user defined named entities. NlpCraft provides wrappers for named-entity recognition components of OpenNLP and Stanford libraries.
+ </li>
+ <li>
+ <code>Entity enrichers</code> optional list. Entity enricher is component which allows to add additional properties to prepared entities. Can be useful for extending existing entity enrichers functionality.
+ </li>
+ <li>
+ <code>Entity mappers</code> optional list. Entity mapper is component which allows to map one set of entities into another after the entities were parsed and enriched. Can be useful for building complex parsers based on existed.
+ </li>
+ <li>
+ <code>Entity validators</code> optional list. Entities validator is user defined component, where prepared entities are inspected and exceptions can be thrown from user code to break user input processing.
+ </li>
+ <li>
+ <code>Variant filter</code>. Optional component which allows filtering detected variants, rejecting undesirable.
+ </li>
+ </ul>
+
<p>
- To use data model it has to be deployed into a data probe.
+ Below example if <code>Model</code> creation. <code>Pipeline</code> is prepared using <code>NCPipelineBuilder</code> class helper.
</p>
- </section>
- <section id="data-probe">
- <h2 class="section-title">Data Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <pre class="brush: scala, highlight: []">
+ val pipeline =
+ new NCPipelineBuilder().
+ withTokenParser(new NCFrTokenParser()).
+ withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
+ withTokenEnricher(new NCFrStopWordsTokenEnricher()).
+ withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
+ build
+ val cfg = NCModelConfig("nlpcraft.lightswitch.fr.ex", "LightSwitch Example Model FR", "1.0")
+
+ val mdl = new NCModelAdapter(cfg, pipeline)
+ </pre>
+
<p>
- Data probe is a light-weight container designed to securely deploy and manage user data models.
- Each probe can deploy and manage multiple models and many probes can be connected to the REST server (or a cluster of REST servers).
- The main purpose of the data probe is to separate data model hosting from managing REST calls from the clients.
- While you would typically have just one REST server, you may have multiple data probes deployed
- in different geo-locations and configured differently.
+ This flexible system allows to create any pipelines on any language. You can collect NlpCraft predefined components, write your own and easy reuse custom components.
</p>
- <p>
- Data probes can be deployed and run anywhere as long as there is an ingress connectivity from the REST server, and are
- typically deployed in DMZ or close to your target data sources: on-premise, in the cloud, etc. Data
- probe uses strong 256-bit encryption and ingress only connectivity for communicating with the REST server.
- </p>
- </section>
- <section id="server">
- <h2 class="section-title">REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server (or a cluster of REST servers behind a load balancer) provides URL endpoint for end-user applications
- to securely query data sources using natural language via data models deployed in data probes. Its main purpose is to
- accept REST-over-HTTP calls from end-user applications and route these requests to and from requested data probes.
- </p>
- <p>
- Unlike data probe that gets restarted every time the model is changed, i.e. during development, the
- REST server is a "fire-and-forget" component that can be launched once while various data probes can
- continuously reconnect to it. It can typically run as a Docker image locally on premise or in the cloud.
- </p>
- <p>
- Learn more about <a href="data-model.html">data model</a>,
- <a href="server-and-probe.html#probe">data probe</a> and <a href="server-and-probe.html#server">REST server</a>.
- </p>
- </section>
- <section id="in-depth">
- <h2 class="section-title">In-Depth Look <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Watch this full video (34:42) of the presentation from
- <a target=_ href="https://www.apachecon.com/acasia2021/">ApacheCon Asia 2021</a> conference to get in-depth understanding of
- the reasons why NLPCraft project was developed and what are the key principles that underlying it:
- </p>
- <div>
- <iframe
- width="514"
- height="289"
- src="https://www.youtube.com/embed/O7iK0AXvcJ8?modestbranding=1"
- title="NLPCraft - Breaking Years Of Dogma In NLP"
- frameborder="0"
- allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
- allowfullscreen>
- </iframe>
- </div>
</section>
</div>
<div class="col-md-2 third-column">
<ul class="side-nav">
<li class="side-nav-title">On This Page</li>
<li><a href="#overview">Overview</a></li>
- <li><a href="#data-model">Data Model</a></li>
- <li><a href="#data-probe">Data Probe</a></li>
- <li><a href="#server">REST Server</a></li>
{% include quick-links.html %}
</ul>
</div>
diff --git a/examples/alarm_clock.html b/examples/alarm_clock.html
deleted file mode 100644
index a85af49..0000000
--- a/examples/alarm_clock.html
+++ /dev/null
@@ -1,417 +0,0 @@
----
-active_crumb: Alarm Clock <code><sub>ex</sub></code>
-layout: documentation
-id: alarm_clock
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This simple example provides an "alarm clock" implementation where you can ask to set the timer
- for a specific duration from now expressed in hours, minutes and/or seconds. You can say "ping me in 3 minutes",
- "buzz me in an hour and 15 minutes", or "set my alarm for 30 secs". When the timer is up it will
- simply print out "BEEP BEEP BEEP" in the data probe console.
- </p>
- <p>
- <b>Complexity:</b> <span class="complexity-one-star"><i class="fas fa-star"></i> <i class="far fa-star"></i> <i class="far fa-star"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/alarm">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="new_project">
- <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can create new Java project in many ways - we'll use <a href="/tools/script.html">NLPCraft CLI</a>
- to accomplish this task:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-prj-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-prj-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-prj-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=AlarmClock --outputDir=~ --pkgName=demo --mdlType=json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/AlarmClock</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Java and Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-prj-out" role="tabpanel">
- <p></p>
- <img alt="" class="img-fluid" src="/images/alarm_clock_fig1.png">
- </div>
- </div>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to start with declaring the static part of our model using JSON which we will later load using
- <code>NCModelFileAdapter</code> in our Java-based model implementation. Open <code>src/main/resources/<b>alarm_clock.json</b></code>
- file and replace its content with this JSON:
- </p>
- <pre class="brush: js, highlight: [7, 11, 20]">
-{
- "id": "nlpcraft.alarm.ex",
- "name": "Alarm Example Model",
- "version": "1.0",
- "description": "Alarm example model.",
- "enabledBuiltInTokens": [
- "nlpcraft:num"
- ],
- "elements": [
- {
- "id": "x:alarm",
- "description": "Alarm token indicator.",
- "synonyms": [
- "{ping|buzz|wake|call|hit} {me|up|me up|_}",
- "{set|_} {my|_} {wake|wake up|_} {alarm|timer|clock|buzzer|call} {up|_}"
- ]
- }
- ],
- "intents": [
- "intent=alarm term~{# == 'x:alarm'} term(nums)~{# == 'nlpcraft:num' && meta_token('nlpcraft:num:unittype') == 'datetime' && meta_token('nlpcraft:num:isequalcondition') == true}[0,7]"
- ]
-}
- </pre>
- <p>There are number of important points here:</p>
- <ul>
- <li>
- <code>Line 7</code> indicates that we'll be using built-in token <code>nlpcraft:num</code> (and therefore
- it needs to be explicitly enabled).
- </li>
- <li>
- <code>Line 11</code> defines the only custom model element we'll need. It's ID is <code>x:alarm</code> and
- it is defined through synonyms. It basically means a verb to set up an alarm clock.
- </li>
- <li>
- On <code>line 20</code> we define an intent <code>alarm</code> that we are going to be looking for in the user input that
- consists of two terms: one for <code>x:alarm</code> element (defined above) and another one for up to 7 numeric values
- of unit type <code>datetime</code> (minutes, seconds, hours, etc.).
- </li>
- </ul>
- <p>
- Now that our model is ready let's create a Java class that would load this model and provide the actual
- callback for when the intent <code>alarm</code> is detected in the user input.
- </p>
- </section>
- <section id="code">
- <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Open <code>src/main/java/demo/<b>AlarmClock.java</b></code> file and replace its content with the following code:
- </p>
- <pre class="brush: java, highlight: [10, 18, 22, 27], 82">
-package demo;
-
-import org.apache.nlpcraft.model.*;
-import java.time.*;
-import java.time.format.*;
-import java.util.*;
-
-import static java.time.temporal.ChronoUnit.*;
-
-public class AlarmClock extends NCModelFileAdapter {
- private static final DateTimeFormatter FMT =
- DateTimeFormatter.ofPattern("HH'h' mm'm' ss's'").withZone(ZoneId.systemDefault());
-
- private final Timer timer = new Timer();
-
- public AlarmClock() {
- // Loading the model from the file in the classpath.
- super("alarm_model.json");
- }
-
- @NCIntentRef("alarm")
- @NCIntentSample({
- "Ping me in 3 minutes",
- "Buzz me in an hour and 15mins",
- "Set my alarm for 30s"
- })
- private NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("nums") List<NCToken> numToks
- ) {
- if (ctx.isAmbiguous())
- throw new NCRejection("Not exact match.");
-
- long unitsCnt = numToks.stream().map(tok -> (String)tok.meta("num:unit")).distinct().count();
-
- if (unitsCnt != numToks.size())
- throw new NCRejection("Ambiguous time units.");
-
- LocalDateTime now = LocalDateTime.now();
- LocalDateTime dt = now;
-
- for (NCToken num : numToks) {
- String unit = num.meta("nlpcraft:num:unit");
-
- // Skip possible fractional to simplify.
- long v = ((Double)num.meta("nlpcraft:num:from")).longValue();
-
- if (v <= 0)
- throw new NCRejection("Value must be positive: " + unit);
-
- switch (unit) {
- case "second": { dt = dt.plusSeconds(v); break; }
- case "minute": { dt = dt.plusMinutes(v); break; }
- case "hour": { dt = dt.plusHours(v); break; }
- case "day": { dt = dt.plusDays(v); break; }
- case "week": { dt = dt.plusWeeks(v); break; }
- case "month": { dt = dt.plusMonths(v); break; }
- case "year": { dt = dt.plusYears(v); break; }
-
- default:
- // It shouldn't be assert, because 'datetime' unit can be extended.
- throw new NCRejection("Unsupported time unit: " + unit);
- }
- }
-
- long ms = now.until(dt, MILLIS);
-
- assert ms >= 0;
-
- timer.schedule(
- new TimerTask() {
- @Override
- public void run() {
- System.out.println(
- "BEEP BEEP BEEP for: " + ctx.getContext().getRequest().getNormalizedText() + ""
- );
- }
- },
- ms
- );
-
- return NCResult.text("Timer set for: " + FMT.format(dt));
- }
-
- @Override
- public void onDiscard() {
- // Clean up when model gets discarded (e.g. during testing).
- timer.cancel();
- }
-}
- </pre>
- <p>
- There's a bit of a logic here that deals mostly with taking multiple numeric values and converting them into
- a single number of milliseconds that the alarm clock needs to be set up for. Let's review it step by step:
- </p>
- <ul>
- <li>
- On <code>line 10</code> our class extends <code>NCModelFileAdapter</code> that allows us to load most
- of the model declaration from the external JSON or YAML file (line 18) and only provide functionality that we
- couldn't express in declarative portion in JSON.
- </li>
- <li>
- <code>Line 27</code> defines method <code>onMatch</code> as a callback for intent <code>alarm</code>
- when it is detected in the user input. Method parameter <code>numToks</code> will get up to 7 tokens
- of type <code>nlpcraft:num</code> (see intent definition above).
- </li>
- <li>
- Note the <code>line 22</code> where we use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation to provide samples of the user input that this intent should match. Apart from documentation
- purpose these samples will be used when we will be <a href="#testing">testing out model below.</a>
- </li>
- <li>
- The rest of the method <code>onMatch</code> implementation is a relatively straight forward Java code
- that calculates timer delay from multiple numeric units and their types. In the end (line 68)
- it schedules a timer to print "BEEP BEEP BEEP" at calculated time. For simplicity, this message will
- be printed right in the data probe console.
- </li>
- <li>
- On the <code>line 82</code> the intent callback simply returns a confirmation message telling
- for what actual time the alarm clock was set.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Once we have our model ready let's go to <code>~/AlarmClock</code> directory and run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/AlarmClock
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component that you generally needs to start only once for this and any other
- examples.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Remember the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our model code next to intent definition?
- </p>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/AlarmClock/target/classes --mdls=demo.AlarmClock
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- <p>
- Look at the output of this command and you will see the test results for all our sample utterances:
- </p>
- <p>
- <img style="max-width: 667px !important;" class="img-fluid" alt="" src="/images/alarm-clock-test.png">
- </p>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created alarm clock data model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#new_project">New Project</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#code">Model Class</a></li>
- <li><a href="#build_project">Build Project</li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/examples/calculator.html b/examples/calculator.html
new file mode 100644
index 0000000..2d95088
--- /dev/null
+++ b/examples/calculator.html
@@ -0,0 +1,265 @@
+---
+active_crumb: Calculator <code><sub>ex</sub></code>
+layout: documentation
+id: calculator
+fa_icon: fa-cube
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column example">
+ <section id="overview">
+ <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ This example provides a very simple calculator implementation.
+ It supports restricted set of arithmetic operations for numeric values.
+ </p>
+ <p>
+ <b>Complexity:</b> <span class="complexity-one-star"><i class="fas fa-star"></i> <i class="far fa-star"></i> <i class="far fa-star"></i></span><br/>
+ <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/caclulator">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
+ <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
+ </p>
+ </section>
+ <section id="new_project">
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
+ </p>
+ <pre class="brush: js, highlight: []">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft Calculator Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
+ </pre>
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>CalculatorModel.scala</code> - Scala class, model implementation.</li>
+ <li><code>CalculatorModelSpec.scala</code> - Scala tests class, which allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [8, 12]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | \--scala
+ | \--demo
+ | CalculatorModel.scala
+ \--test
+ \--scala
+ \--demo
+ CalculatorModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ All element definitions are provided programmatically inside Scala model <code>CalculatorModel</code> class.
+ Open <code>src/main/scala/demo/<b>CalculatorModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [11, 12, 14, 21, 22, 23, 31, 35, 41, 47, 56, 61]">
+ package demo
+
+ import edu.stanford.nlp.pipeline.StanfordCoreNLP
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
+ import org.apache.nlpcraft.nlp.entity.parser.stanford.*
+ import org.apache.nlpcraft.nlp.parsers.*
+ import org.apache.nlpcraft.nlp.token.parser.stanford.NCStanfordNLPTokenParser
+ import java.util.Properties
+
+ object CalculatorModel:
+ private val OPS: Map[String, (Int, Int) => Int] =
+ Map("+" -> (_ + _), "-" -> (_ - _), "*" -> (_ * _), "/" -> (_ / _))
+ private val PIPELINE: NCPipeline =
+ val props = new Properties()
+ props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner")
+
+ val stanford = new StanfordCoreNLP(props)
+
+ new NCPipelineBuilder().
+ withTokenParser(new NCStanfordNLPTokenParser(stanford)).
+ withEntityParser(new NCNLPEntityParser(t => OPS.contains(t.getText))).
+ withEntityParser(new NCStanfordNLPEntityParser(stanford, Set("number"))).
+ build
+
+ private def nne(e: NCEntity): Int =
+ java.lang.Double.parseDouble(e[String]("stanford:number:nne")).intValue
+
+ import CalculatorModel.*
+
+ class CalculatorModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.calculator.ex", "Calculator Example Model", "1.0"),
+ PIPELINE
+ ) :
+ private var mem: Option[Int] = None
+
+ private def calc(x: Int, op: String, y: Int): NCResult =
+ mem = Some(OPS.getOrElse(op, throw new IllegalStateException()).apply(x, y))
+ NCResult(mem.get)
+
+ @NCIntent(
+ "intent=calc options={ 'ordered': true }" +
+ " term(x)={# == 'stanford:number'}" +
+ " term(op)={has(list('+', '-', '*', '/'), meta_ent('nlp:token:text')) == true}" +
+ " term(y)={# == 'stanford:number'}"
+ )
+ def onMatch(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("x") x: NCEntity,
+ @NCIntentTerm("op") op: NCEntity,
+ @NCIntentTerm("y") y: NCEntity
+ ): NCResult =
+ calc(nne(x), op.mkText, nne(y))
+
+ @NCIntent(
+ "intent=calcMem options={ 'ordered': true }" +
+ " term(op)={has(list('+', '-', '*', '/'), meta_ent('nlp:token:text')) == true}" +
+ " term(y)={# == 'stanford:number'}"
+ )
+ def onMatchMem(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("op") op: NCEntity,
+ @NCIntentTerm("y") y: NCEntity
+ ): NCResult =
+ calc(mem.getOrElse(throw new NCRejection("Memory is empty.")), op.mkText, nne(y))
+ </pre>
+ <p>
+ There are two intents with simple logic. First returns arithmetic operation result with two input parameters,
+ second uses last operation result instead of the first argument. Note also that the arithmetic notations
+ (<code>+</code>, <code>-</code>, <code>*</code>, <code>/</code>) are used as is for without any additional synonyms.
+ Let's review this implementation step by step:
+ </p>
+ <ul>
+ <li>
+ <code>Line 11</code> declares <code>CalculatorModel</code>, model companion object, which contains
+ static content and helper methods.
+ </li>
+ <li>
+ <code>Line 12</code> defines arithmetic operations map with notations as keys and functions definitions as values.
+ </li>
+ <li>
+ <code>Line 14</code> defines model pipeline based on three built components.
+ <code>Line 21</code> defines Stanford token parser <code>NCStanfordNLPTokenParser</code>
+ (we use Stanford NLP components in this example).
+ </li>
+ <li>
+ <code>Line 22</code> declares entity parser <code>NCNLPEntityParser</code>
+ which allows to find arithmetic operations notations.
+ </li>
+ <li>
+ <code>Line 23</code> defines entity parser <code>NCStanfordNLPEntityParser</code>,
+ which allows to find numerics in the text input.
+ </li>
+ <li>
+ <code>Line 31</code> declares<code>CalculatorModel</code> model class.
+ </li>
+ <li>
+ <code>line 35</code> declares variable named <code>mem</code> which act as a holder for the last operation result.
+ </li>
+ <li>
+ <code>Lines 41 and 47</code> annotate intents <code>calc</code> and its callback method <code>onMatch()</code>.
+ Intent <code>calc</code> requires one arithmetic operation notation and two numerics as its arguments.
+ </li>
+ <li>
+ <code>Lines 56 and 61</code> annotate intents <code>calcMem</code> and its callback method <code>onMatchMem()</code>.
+ Intent <code>calcMem</code> requires one arithmetic operation notation and one numeric as its second arguments.
+ Note that it attempts to use last operation result as its first argument, if one exists.
+ </li>
+ </ul>
+ </section>
+
+ <section id="testing">
+ <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ The test provided in <code>CalculatorModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intents <code>calc</code> or <code>calcMem</code>:
+ </p>
+ <pre class="brush: scala, highlight: [9, 10, 15, 16]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class CalculatorModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new CalculatorModel())) { client =>
+ def check(txt: String, v: Int): Unit =
+ require(v == client.ask(txt, "userId").getBody)
+
+ check("2 + 2", 4)
+ check("3 * 4", 12)
+ check("/ two", 6)
+ check("+ twenty two", 28)
+ check("7 + 2", 9)
+ }
+ }
+ </pre>
+ <ul>
+ <li>
+ On <code>line 9</code> the client for our model is created.
+ </li>
+ <li>
+ On <code>line 10</code> the method <code>ask()</code> is called. Its result is checked with expected value.
+ </li>
+ <li>
+ Note that test sentences on <code>lines 15, 16</code> trigger <code>calcMem</code> intent while other sentences
+ trigger <code>calc</code> intent.
+ </li>
+ </ul>
+ <p>
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
+ </p>
+ <pre class="brush: scala, highlight: []">
+ $ sbt executeTests
+ </pre>
+ </section>
+ <section>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You've created calculator model and tested it.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#new_project">New Project</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#testing">Testing</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
+
+
diff --git a/examples/light_switch.html b/examples/light_switch.html
index 81fbaa3..aea3015 100644
--- a/examples/light_switch.html
+++ b/examples/light_switch.html
@@ -26,10 +26,10 @@
<section id="overview">
<h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- This example provides very simple implementation for NLI-powered light switch. You can say something like
+ This example provides a very simple implementation for NLI-powered light switch. You can say something like
"Turn the lights off in the entire house" or "Switch on the illumination in the master bedroom closet".
- You can modify intent callbacks to perform the actual light switching using HomeKit or Arduino-based
- controllers.
+ By modifying intent callbacks using, for example, HomeKit or Arduino-based controllers you can provide the
+ actual light switching.
</p>
<p>
<b>Complexity:</b> <span class="complexity-one-star"><i class="fas fa-star"></i> <i class="far fa-star"></i> <i class="far fa-star"></i></span><br/>
@@ -40,185 +40,139 @@
<section id="new_project">
<h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- You can create new Scala projects in many ways - we'll use <a href="/tools/script.html">NLPCraft CLI</a>
- to accomplish this task:
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
</p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-prj-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-prj-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-prj-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=LightSwitch --outputDir=~ --pkgName=demo --lang=scala
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/LightSwitch</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Java and Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-prj-out" role="tabpanel">
- <p></p>
- <img alt="" class="img-fluid" src="/images/light_switch_fig1.png">
- </div>
- </div>
+ <pre class="brush: js, highlight: []">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft LightSwitch Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
+ </pre>
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>lightswitch_model.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>LightSwitchModel.scala</code> - Scala class, model implementation.</li>
+ <li><code>LightSwitchModelSpec.scala</code> - Scala tests class, which allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 10, 14]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | lightswitch_model.yaml
+ | \--scala
+ | \--demo
+ | LightSwitchModel.scala
+ \--test
+ \--scala
+ \--demo
+ LightSwitchModelSpec.scala
+ </pre>
</section>
<section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
We are going to start with declaring the static part of our model using YAML which we will later load using
- <code>NCModelFileAdapter</code> in our Scala-based model implementation. Open <code>src/main/resources/<b>light_switch.yaml</b></code>
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>light_switch.yaml</b></code>
file and replace its content with the following YAML:
</p>
- <pre class="brush: js, highlight: [6, 22, 23, 26, 33, 41, 52]">
-id: "nlpcraft.lightswitch.ex"
-name: "Light Switch Example Model"
-version: "1.0"
-description: "NLI-powered light switch example model."
-
-macros:
- - name: "<ACTION>"
- macro: "{turn|switch|dial|let|set|get|put}"
- - name: "<KILL>"
- macro: "{shut|kill|stop|eliminate}"
- - name: "<ENTIRE_OPT>"
- macro: "{entire|full|whole|total|_}"
- - name: "<FLOOR_OPT>"
- macro: "{upstairs|downstairs|{1st|2nd|3rd|4th|5th|top|ground} floor|_}"
- - name: "<TYPE>"
- macro: "{room|closet|attic|loft|{store|storage} {room|_}}"
- - name: "<LIGHT>"
- macro: "{all|_} {it|them|light|illumination|lamp|lamplight}"
-
-enabledBuiltInTokens: []
+ <pre class="brush: js, highlight: [1, 10, 17, 25]">
+ macros:
+ "<ACTION>" : "{turn|switch|dial|let|set|get|put}"
+ "<KILL>" : "{shut|kill|stop|eliminate}"
+ "<ENTIRE_OPT>" : "{entire|full|whole|total|_}"
+ "<FLOOR_OPT>" : "{upstairs|downstairs|{1st|first|2nd|second|3rd|third|4th|fourth|5th|fifth|top|ground} floor|_}"
+ "<TYPE>" : "{room|closet|attic|loft|{store|storage} {room|_}}"
+ "<LIGHT>" : "{all|_} {it|them|light|illumination|lamp|lamplight}"
-permutateSynonyms: true
-sparse: true
+ elements:
+ - id: "ls:loc"
+ description: "Location of lights."
+ synonyms:
+ - "<ENTIRE_OPT> <FLOOR_OPT> {kitchen|library|closet|garage|office|playroom|{dinning|laundry|play} <TYPE>}"
+ - "<ENTIRE_OPT> <FLOOR_OPT> {master|kid|children|child|guest|_} {bedroom|bathroom|washroom|storage} {<TYPE>|_}"
+ - "<ENTIRE_OPT> {house|home|building|{1st|first} floor|{2nd|second} floor}"
-elements:
- - id: "ls:loc"
- description: "Location of lights."
- synonyms:
- - "<ENTIRE_OPT> <FLOOR_OPT> {kitchen|library|closet|garage|office|playroom|{dinning|laundry|play} <TYPE>}"
- - "<ENTIRE_OPT> <FLOOR_OPT> {master|kid|children|child|guest|_} {bedroom|bathroom|washroom|storage} {<TYPE>|_}"
- - "<ENTIRE_OPT> {house|home|building|{1st|first} floor|{2nd|second} floor}"
+ - id: "ls:on"
+ groups:
+ - "act"
+ description: "Light switch ON action."
+ synonyms:
+ - "<ACTION> {on|up|_} <LIGHT> {on|up|_}"
+ - "<LIGHT> {on|up}"
- - id: "ls:on"
- groups:
- - "act"
- description: "Light switch ON action."
- synonyms:
- - "<ACTION> {on|up|_} <LIGHT> {on|up|_}"
- - "<LIGHT> {on|up}"
-
- - id: "ls:off"
- groups:
- - "act"
- description: "Light switch OFF action."
- synonyms:
- - "<ACTION> <LIGHT> {off|out}"
- - "{<ACTION>|<KILL>} {off|out} <LIGHT>"
- - "<KILL> <LIGHT>"
- - "<LIGHT> <KILL>"
- - "no <LIGHT>"
-
-intents:
- - "intent=ls term(act)={has(tok_groups, 'act')} term(loc)={# == 'ls:loc'}*"
+ - id: "ls:off"
+ groups:
+ - "act"
+ description: "Light switch OFF action."
+ synonyms:
+ - "<ACTION> <LIGHT> {off|out|down}"
+ - "{<ACTION>|<KILL>} {off|out|down} <LIGHT>"
+ - "<KILL> <LIGHT>"
+ - "<LIGHT> <KILL>"
+ - "{out|no|off|down} <LIGHT>"
+ - "<LIGHT> {out|off|down}"
</pre>
<p>There are number of important points here:</p>
<ul>
<li>
- <code>Line 6</code> defines several macros that are used later on throughout the model's elements
+ <code>Line 1</code> defines several macros that are used later on throughout the model's elements
to shorten the synonym declarations. Note how macros coupled with option groups
shorten overall synonym declarations 1000:1 vs. manually listing all possible word permutations.
</li>
<li>
- <code>Lines 22, 23</code> define model properties that allow for multi-word synonyms in this model
- to be
- <a class="not-code" target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelView.html#isSparse()">sparse</a> and
- <a class="not-code" target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCModelView.html#isPermutateSynonyms()">permutate</a> them for better detection. These two properties generally enable a free-form
- natural language comprehension.
- </li>
- <li>
- <code>Lines 26, 33, 41</code> define three model elements: the location of the light, and actions to turn
+ <code>Lines 10, 17, 25</code> define three model elements: the location of the light, and actions to turn
the light on and off. Action elements belong to the same group <code>act</code> which
- will be used in our intent (<code>line 42</code>). Note that these model elements are defined mostly
- through macros we have provided above.
- </li>
- <li>
- On <code>line 52</code> we define a non-conversational intent <code>ls</code> that requires
- one action (a token belonging to the group <code>act</code>) and optional list of light locations
- (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ will be used in our intent, defined in <code>LightSwitchModel</code> class. Note that these model
+ elements are defined mostly through macros we have defined above.
</li>
</ul>
- <p>
- Now that our model is ready let's create a Java class that would load this model and provide the actual
- callback for when the intent <code>ls</code> is detected in the user input.
- </p>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>LightSwitchModel</code> class as well.
+ </p>
+ </div>
</section>
<section id="code">
<h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- Open <code>src/main/scala/demo/<b>LightSwitch.scala</b></code> file and replace its content with the following code:
+ Open <code>src/main/scala/demo/<b>LightSwitchModel.scala</b></code> file and replace its content with the following code:
</p>
- <pre class="brush: scala, highlight: [5, 6, 7, 25, 26, 38]">
-package demo
+ <pre class="brush: scala, highlight: [6, 7, 8, 10, 11, 14, 15, 22]">
+ package demo
-import org.apache.nlpcraft.model.{NCIntentTerm, _}
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
-class LightSwitch extends NCModelFileAdapter("light_switch.yaml") {
- @NCIntentRef("ls")
- @NCIntentSample(Array(
- "Turn the lights off in the entire house.",
- "Turn off all lights now",
- "Switch on the illumination in the master bedroom closet.",
- "Get the lights on.",
- "Lights up in the kitchen.",
- "Please, put the light out in the upstairs bedroom.",
- "Set the lights on in the entire house.",
- "Turn the lights off in the guest bedroom.",
- "Could you please switch off all the lights?",
- "Dial off illumination on the 2nd floor.",
- "Please, no lights!",
- "Kill off all the lights now!",
- "No lights in the bedroom, please.",
- "Light up the garage, please!",
- "Kill the illumination now!"
- ))
- def onMatch(
- @NCIntentTerm("act") actTok: NCToken,
- @NCIntentTerm("loc") locToks: List[NCToken]
- ): NCResult = {
- val status = if (actTok.getId == "ls:on") "on" else "off"
- val locations =
- if (locToks.isEmpty)
- "entire house"
- else
- locToks.map(_.getOriginalText()).mkString(", ")
+ class LightSwitchModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.lightswitch.java.ex", "LightSwitch Example Model", "1.0"),
+ new NCPipelineBuilder().withSemantic("en", "lightswitch_model.yaml").build
+ ):
+ @NCIntent("intent=ls term(act)={has(ent_groups, 'act')} term(loc)={# == 'ls:loc'}*")
+ def onMatch(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("act") actEnt: NCEntity,
+ @NCIntentTerm("loc") locEnts: List[NCEntity]
+ ): NCResult =
+ val status = if actEnt.getId == "ls:on" then "on" else "off"
+ val locations = if locEnts.isEmpty then "entire house" else locEnts.map(_.mkText).mkString(", ")
- // Add HomeKit, Arduino or other integration here.
-
- // By default - return a descriptive action string.
- NCResult.text(s"Lights '$status' in '${locations.toLowerCase}'.")
- }
-}
+ // Add HomeKit, Arduino or other integration here.=
+ // By default - just return a descriptive action string.
+ NCResult(s"Lights are [$status] in [${locations.toLowerCase}].")
</pre>
<p>
The intent callback logic is very simple - we return a descriptive confirmation message
@@ -227,160 +181,100 @@
</p>
<ul>
<li>
- On <code>line 5</code> our class extends <code>NCModelFileAdapter</code> that allows us to load most
- of the model declaration from the external YAML file and only provide functionality that we
- couldn't express in declarative portion in YAML.
+ On <code>line 5</code> our class extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
</li>
<li>
- <code>Line 6</code> annotates method <code>onMatch</code> as a callback for the intent <code>ls</code>
- when it is detected in the user input. Note that intent <code>ls</code> is defined in the <code>light_switch.yaml</code>
- file that was loaded by <code>NCModelFileAdapter</code> class.
+ On <code>line 6</code> created model configuration with most default parameters.
</li>
<li>
- Note the <code>line 7</code> where we use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation to provide samples of the user input that this intent should match. Apart from documentation
- purpose these samples will be used when we will be <a href="#testing">testing out model below.</a>
+ On <code>line 7</code> created pipeline, based on semantic model definition,
+ described in <code>lightswitch_model.yaml</code> file.
</li>
<li>
- <code>Lines 25 and 26</code> map terms from detected intent to the formal method parameters of the
+ <code>Lines 10 and 11</code> annotates intents <code>ls</code> and its callback method <code>onMatch</code>.
+ Intent <code>ls</code> requires one action (a token belonging to the group <code>act</code>) and optional list of light locations
+ (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ </li>
+ <li>
+ <code>Lines 14 and 15</code> map terms from detected intent to the formal method parameters of the
<code>onMatch</code> method.
</li>
<li>
- On the <code>line 38</code> the intent callback simply returns a confirmation message.
+ On the <code>line 22</code> the intent callback simply returns a confirmation message.
</li>
</ul>
</section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Once we have our model ready let's go to <code>~/LightSwitch</code> directory and run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/LightSwitch
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component that you generally need to start it only once
- for this and other examples.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
+
<section id="testing">
<h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- Remember the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our model code next to intent definition?
+ The test defined in <code>LightSwitchModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intent <code>ls</code>:
</p>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/LightSwitch/target/classes --mdls=demo.LightSwitch
+ <pre class="brush: scala, highlight: [9, 11]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class LightSwitchModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new LightSwitchModel())) { client =>
+ def check(txt: String): Unit =
+ require(client.debugAsk(txt, "userId", true).getIntentId == "ls")
+
+ check("Turn the lights off in the entire house.")
+ check("Turn off all lights now")
+ check("Switch on the illumination in the master bedroom closet.")
+ check("Get the lights on.")
+ check("Off the lights on the 1st floor")
+ check("Lights up in the kitchen.")
+ check("Please, put the light out in the upstairs bedroom.")
+ check("Set the lights on in the entire house.")
+ check("Turn the lights off in the guest bedroom.")
+ check("Could you please switch off all the lights?")
+ check("Dial off illumination on the 2nd floor.")
+ check("Turn down lights in 1st floor bedroom")
+ check("Lights on at second floor kitchen")
+ check("Please, no lights!")
+ check("Kill off all the lights now!")
+ check("Down the lights in the garage")
+ check("Lights down in the kitchen!")
+ check("Turn up the illumination in garage and master bedroom")
+ check("Turn down all the light now!")
+ check("No lights in the bedroom, please.")
+ check("Light up the garage, please!")
+ check("Kill the illumination now!")
+ }
+ }
</pre>
- <p>
- <b>NOTES:</b>
- </p>
<ul>
<li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
+ On <code>line 9</code> the client for our model is created.
</li>
<li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
+ On <code>line 11</code> a special method <code>debugAsk</code> is called.
+ It allows to check the winning intent and its callback parameters without actually
+ calling the intent.
</li>
<li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
+ <code>Lines 13-34</code> define all the test input sentences that should all
+ trigger <code>ls</code> intent.
</li>
</ul>
<p>
- Look at the output of this command and you will see the test results for all our sample utterances:
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
</p>
- <p>
- <img style="max-width: 928px !important;" class="img-fluid" alt="" src="/images/light-switch-test.png">
- </p>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
+ <pre class="brush: scala, highlight: []">
+ $ sbt executeTests
+ </pre>
</section>
<section>
<h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
- You've created light switch data model, started the REST server and tested this model using the built-in test framework.
+ You've created light switch data model and tested it.
</p>
</section>
</div>
@@ -391,10 +285,7 @@
<li><a href="#new_project">New Project</a></li>
<li><a href="#model">Data Model</a></li>
<li><a href="#code">Model Class</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#start_server">Start Server</a></li>
<li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
{% include quick-links.html %}
</ul>
</div>
diff --git a/examples/light_switch_fr.html b/examples/light_switch_fr.html
new file mode 100644
index 0000000..d3d0047
--- /dev/null
+++ b/examples/light_switch_fr.html
@@ -0,0 +1,491 @@
+---
+active_crumb: Light Switch FR <code><sub>ex</sub></code>
+layout: documentation
+id: light_switch_fr
+fa_icon: fa-cube
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column example">
+ <section id="overview">
+ <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ This example provides a very simple French language implementation for NLI-powered light switch. You can say something like
+ "Éteignez les lumières dans toute la maison" or "Allumez les lumières".
+ By modifying intent callbacks using, for example, HomeKit or Arduino-based controllers you can provide the
+ actual light switching.
+ </p>
+ <p>
+ <b>Complexity:</b> <span class="complexity-two-star"><i class="fas fa-square"></i> <i class="fas fa-square"></i> <i class="far fa-square"></i></span><br/>
+ <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/lightswitch_fr">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
+ <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
+ </p>
+ </section>
+ <section id="new_project">
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
+ </p>
+ <pre class="brush: js, highlight: [8, 9, 10]">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft LightSwitch FR Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.apache.lucene" % "lucene-analyzers-common" % "8.11.2",
+ libraryDependencies += "org.languagetool" % "languagetool-core" % "5.8",
+ libraryDependencies += "org.languagetool" % "language-fr" % "5.8"
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
+ </pre>
+
+ <p>
+ On <code>lines 8, 9 and 10</code> added libraries, which used for support base NLP operations with French language.
+ </p>
+
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>lightswitch_model_fr.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>LightSwitchFrModel.scala</code> - Model implementation.</li>
+ <li><code>NCFrSemanticEntityParser.scala</code> - Semantic entity parser, custom implementation for French language.</li>
+ <li><code>NCFrLemmaPosTokenEnricher.scala</code> - Lemma and point of speech token enricher, custom implementation for French language.</li>
+ <li><code>NCFrStopWordsTokenEnricher.scala</code> - Stop-words token enricher, custom implementation for French language.</li>
+ <li><code>NCFrTokenParser.scala</code> - Token parser, custom implementation for French language.</li>
+ <li><code>LightSwitchFrModelSpec.scala</code> - Test that allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 10, 14, 17, 18, 20, 24]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | lightswitch_model_fr.yaml
+ | \--scala
+ | \--demo
+ | | LightSwitchFrModel.scala
+ | \--nlp
+ | +--entity
+ | | \--parser
+ | | NCFrSemanticEntityParser.scala
+ | \--token
+ | +--enricher
+ | | NCFrLemmaPosTokenEnricher.scala
+ | | NCFrStopWordsTokenEnricher.scala
+ | \--parser
+ | NCFrTokenParser.scala
+ \--test
+ \--scala
+ \--demo
+ LightSwitchFrModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ We are going to start with declaring the static part of our model using YAML which we will later load using
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>light_switch_fr.yaml</b></code>
+ file and replace its content with the following YAML:
+ </p>
+ <pre class="brush: js, highlight: [1, 10, 17, 25]">
+ macros:
+ "<ACTION>" : "{allumer|laisser|mettre}"
+ "<KILL>" : "{éteindre|couper|tuer|arrêter|éliminer|baisser|no}"
+ "<ENTIRE_OPT>" : "{entière|pleine|tout|total|_}"
+ "<FLOOR_OPT>" : "{là-haut|à l'étage|en bas|{1er|premier|2ème|deuxième|3ème|troisième|4ème|quatrième|5ème|cinquième|dernier|haut|rez-de-chaussée|en bas} étage|_}"
+ "<TYPE>" : "{chambre|salle|pièce|placard|mansardé|loft|mezzanine|rangement {chambre|salle|pièce|_}}"
+ "<LIGHT>" : "{tout|_} {cela|lumière|éclairage|illumination|lampe}"
+
+ elements:
+ - id: "ls:loc"
+ description: "Location of lights."
+ synonyms:
+ - "<ENTIRE_OPT> <FLOOR_OPT> {cuisine|bibliothèque|placard|garage|bureau|salle de jeux|{salle à manger|buanderie|jeu} <TYPE>}"
+ - "<ENTIRE_OPT> <FLOOR_OPT> {maître|gamin|bébé|enfant|hôte|client|_} {coucher|bains|toilette|rangement} {<TYPE>|_}"
+ - "<ENTIRE_OPT> {maison|foyer|bâtiment|{1er|premier} étage|chaussée|{2ème|deuxième} étage}"
+
+ - id: "ls:on"
+ groups:
+ - "act"
+ description: "Light switch ON action."
+ synonyms:
+ - "{<ACTION>|_} <LIGHT>"
+ - "{<LIGHT>|_} <ACTION>"
+
+ - id: "ls:off"
+ groups:
+ - "act"
+ description: "Light switch OFF action."
+ synonyms:
+ - "<KILL> <LIGHT>"
+ - "<LIGHT> <KILL>"
+ </pre>
+ <p>There are number of important points here:</p>
+ <ul>
+ <li>
+ <code>Line 1</code> defines several macros that are used later on throughout the model's elements
+ to shorten the synonym declarations. Note how macros coupled with option groups
+ shorten overall synonym declarations 1000:1 vs. manually listing all possible word permutations.
+ </li>
+ <li>
+ <code>Lines 10, 17, 25</code> define three model elements: the location of the light, and actions to turn
+ the light on and off. Action elements belong to the same group <code>act</code> which
+ will be used in our intent, defined in <code>LightSwitchFrModel</code> class. Note that these model
+ elements are defined mostly through macros we have defined above.
+
+ </li>
+ </ul>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>LightSwitchFrModel</code> class as well.
+ </p>
+ </div>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Open <code>src/main/scala/demo/<b>LightSwitchFrModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [11, 12, 13, 20, 21, 24, 25, 32]">
+ package demo
+
+ import com.google.gson.Gson
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
+ import demo.nlp.entity.parser.NCFrSemanticEntityParser
+ import demo.lightswitch.nlp.token.enricher.*
+ import demo.lightswitch.nlp.token.parser.NCFrTokenParser
+ import scala.jdk.CollectionConverters.*
+
+ class LightSwitchFrModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.lightswitch.fr.ex", "LightSwitch Example Model FR", "1.0"),
+ new NCPipelineBuilder().
+ withTokenParser(new NCFrTokenParser()).
+ withTokenEnricher(new NCFrLemmaPosTokenEnricher()).
+ withTokenEnricher(new NCFrStopWordsTokenEnricher()).
+ withEntityParser(new NCFrSemanticEntityParser("lightswitch_model_fr.yaml")).
+ build
+ ):
+ @NCIntent("intent=ls term(act)={has(ent_groups, 'act')} term(loc)={# == 'ls:loc'}*")
+ def onMatch(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("act") actEnt: NCEntity,
+ @NCIntentTerm("loc") locEnts: List[NCEntity]
+ ): NCResult =
+ val action = if actEnt.getId == "ls:on" then "allumer" else "éteindre"
+ val locations = if locEnts.isEmpty then "toute la maison" else locEnts.map(_.mkText).mkString(", ")
+
+ // Add HomeKit, Arduino or other integration here.
+ // By default - just return a descriptive action string.
+ NCResult(new Gson().toJson(Map("locations" -> locations, "action" -> action).asJava))
+ </pre>
+ <p>
+ The intent callback logic is very simple - we return a descriptive confirmation message
+ back (explaining what lights were changed). With action and location detected, you can add
+ the actual light switching using HomeKit or Arduino devices. Let's review this implementation step by step:
+ </p>
+ <ul>
+ <li>
+ On <code>line 11</code> our class extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
+ </li>
+ <li>
+ <code>Line 12</code> creates model configuration with most default parameters.
+ </li>
+ <li>
+ <code>Line 13</code> creates pipeline based on custom French language components:
+ <ul>
+ <li><code>NCFrTokenParser</code> - Token parser.</li>
+ <li><code>NCFrLemmaPosTokenEnricher</code> - Lemma and point of speech token enricher.</li>
+ <li><code>NCFrStopWordsTokenEnricher</code> - Stop-words token enricher.</li>
+ <li><code>NCFrSemanticEntityParser</code> - Semantic entity parser extending.</li>
+ </ul>
+ Note that <code>NCFrSemanticEntityParser</code> is based on semantic model definition
+ described in <code>lightswitch_model_fr.yaml</code> file.
+ </li>
+ <li>
+ <code>Lines 20 and 21</code> annotate intents <code>ls</code> and its callback method <code>onMatch()</code>.
+ Intent <code>ls</code> requires one action (a token belonging to the group <code>act</code>) and optional list of light locations
+ (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ </li>
+ <li>
+ <code>Lines 24 and 25</code> map terms from detected intent to the formal method parameters of the
+ <code>onMatch()</code> method.
+ </li>
+ <li>
+ On the <code>line 32</code> the intent callback simply returns a confirmation message.
+ </li>
+ </ul>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Custom Components <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/parser/<b>NCFrTokenParser.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [19]">
+ package demo.nlp.token.parser
+
+ import org.apache.nlpcraft.*
+ import org.languagetool.tokenizers.fr.FrenchWordTokenizer
+ import scala.jdk.CollectionConverters.*
+
+ class NCFrTokenParser extends NCTokenParser:
+ private val tokenizer = new FrenchWordTokenizer
+
+ override def tokenize(text: String): List[NCToken] =
+ val toks = collection.mutable.ArrayBuffer.empty[NCToken]
+ var sumLen = 0
+
+ for ((word, idx) <- tokenizer.tokenize(text).asScala.zipWithIndex)
+ val start = sumLen
+ val end = sumLen + word.length
+
+ if word.strip.nonEmpty then
+ toks += new NCPropertyMapAdapter with NCToken:
+ override def getText: String = word
+ override def getIndex: Int = idx
+ override def getStartCharIndex: Int = start
+ override def getEndCharIndex: Int = end
+
+ sumLen = end
+
+ toks.toList
+ </pre>
+ <ul>
+ <li>
+ <code>NCFrTokenParser</code> is a simple wrapper which implements <code>NCTokenParser</code> based on
+ open source <a href="https://languagetool.org">Language Tool</a> library.
+ </li>
+ <li>
+ On <code>line 19</code> the <code>NCToken</code> instances created.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/enricher/<b>NCFrLemmaPosTokenEnricher.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [27, 28]">
+ package demo.nlp.token.enricher
+
+ import org.apache.nlpcraft.*
+ import org.languagetool.AnalyzedToken
+ import org.languagetool.tagging.fr.FrenchTagger
+ import scala.jdk.CollectionConverters.*
+
+ class NCFrLemmaPosTokenEnricher extends NCTokenEnricher:
+ private def nvl(v: String, dflt : => String): String = if v != null then v else dflt
+
+ override def enrich(req: NCRequest, cfg: NCModelConfig, toks: List[NCToken]): Unit =
+ val tags = FrenchTagger.INSTANCE.tag(toks.map(_.getText).asJava).asScala
+
+ require(toks.sizeIs == tags.size)
+
+ toks.zip(tags).foreach { case (tok, tag) =>
+ val readings = tag.getReadings.asScala
+
+ val (lemma, pos) = readings.size match
+ // No data. Lemma is word as is, POS is undefined.
+ case 0 => (tok.getText, "")
+ // Takes first. Other variants ignored.
+ case _ =>
+ val aTok: AnalyzedToken = readings.head
+ (nvl(aTok.getLemma, tok.getText), nvl(aTok.getPOSTag, ""))
+
+ tok.put("pos", pos)
+ tok.put("lemma", lemma)
+
+ () // Otherwise NPE.
+ }
+ </pre>
+ <ul>
+ <li>
+ <code>NCFrLemmaPosTokenEnricher</code> lemma and point of speech tokens enricher based on
+ open source <a href="https://languagetool.org">Language Tool</a> library.
+ </li>
+ <li>
+ On <code>line 27 and 28</code> the tokens are enriched by <code>pos</code> and <code>lemma</code> data.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/enricher/<b>NCFrStopWordsTokenEnricher.scala</b></code> file and replace its content with the following code:
+ </p>
+
+ <pre class="brush: scala, highlight: [17]">
+ package demo.nlp.token.enricher
+
+ import org.apache.lucene.analysis.fr.FrenchAnalyzer
+ import org.apache.nlpcraft.*
+
+ class NCFrStopWordsTokenEnricher extends NCTokenEnricher:
+ private final val stops = FrenchAnalyzer.getDefaultStopSet
+
+ private def getPos(t: NCToken): String = t.get("pos").getOrElse(throw new NCException("POS not found in token."))
+ private def getLemma(t: NCToken): String = t.get("lemma").getOrElse(throw new NCException("Lemma not found in token."))
+
+ override def enrich(req: NCRequest, cfg: NCModelConfig, toks: List[NCToken]): Unit =
+ for (t <- toks)
+ val lemma = getLemma(t)
+ lazy val pos = getPos(t)
+
+ t.put(
+ "stopword",
+ lemma.length == 1 && !Character.isLetter(lemma.head) && !Character.isDigit(lemma.head) ||
+ stops.contains(lemma.toLowerCase) ||
+ pos.startsWith("I") ||
+ pos.startsWith("O") ||
+ pos.startsWith("P") ||
+ pos.startsWith("D")
+ )
+ </pre>
+ <ul>
+ <li>
+ <code>NCFrStopWordsTokenEnricher</code> is a stop-words tokens enricher based on
+ open source <a href="https://lucene.apache.org/">Apache Lucene</a> library.
+ </li>
+ <li>
+ On <code>line 17</code> tokens are enriched by <code>stopword</code> flags data.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/entity/parser/<b>NCFrSemanticEntityParser.scala</b></code> file and replace its content with the following code:
+ </p>
+
+ <pre class="brush: scala, highlight: [8, 12]">
+ package demo.nlp.entity.parser
+
+ import opennlp.tools.stemmer.snowball.SnowballStemmer
+ import org.apache.nlpcraft.examples.lightswitch.nlp.token.parser.NCFrTokenParser
+ import org.apache.nlpcraft.nlp.parsers.*
+
+ class NCFrSemanticEntityParser(src: String) extends NCSemanticEntityParser(
+ new NCSemanticStemmer:
+ private val stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.FRENCH)
+ override def stem(txt: String): String = stemmer.synchronized { stemmer.stem(txt.toLowerCase).toString }
+ ,
+ new NCFrTokenParser(),
+ mdlSrcOpt = Option(src)
+ )
+ </pre>
+ <ul>
+ <li>
+ <code>NCFrSemanticEntityParser</code> extends <code>NCSemanticEntityParser</code>.
+ It uses stemmer implementation from <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project.
+ </li>
+ </ul>
+ </section>
+
+ <section id="testing">
+ <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ The test defined in <code>LightSwitchFrModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intent <code>ls</code>:
+ </p>
+ <pre class="brush: scala, highlight: [9, 11]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class LightSwitchFrModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new LightSwitchFrModel)) { client =>
+ def check(txt: String): Unit =
+ require(client.debugAsk(txt, "userId", true).getIntentId == "ls")
+
+ check("Éteignez les lumières dans toute la maison.")
+ check("Éteignez toutes les lumières maintenant.")
+ check("Allumez l'éclairage dans le placard de la chambre des maîtres.")
+ check("Éteindre les lumières au 1er étage.")
+ check("Allumez les lumières.")
+ check("Allumes dans la cuisine.")
+ check("S'il vous plait, éteignez la lumière dans la chambre à l'étage.")
+ check("Allumez les lumières dans toute la maison.")
+ check("Éteignez les lumières dans la chambre d'hôtes.")
+ check("Pourriez-vous éteindre toutes les lumières s'il vous plait?")
+ check("Désactivez l'éclairage au 2ème étage.")
+ check("Éteignez les lumières dans la chambre au 1er étage.")
+ check("Lumières allumées à la cuisine du deuxième étage.")
+ check("S'il te plaît, pas de lumières!")
+ check("Coupez toutes les lumières maintenant!")
+ check("Éteindre les lumières dans le garage.")
+ check("Lumières éteintes dans la cuisine!")
+ check("Augmentez l'éclairage dans le garage et la chambre des maîtres.")
+ check("Baissez toute la lumière maintenant!")
+ check("Pas de lumières dans la chambre, s'il vous plait.")
+ check("Allumez le garage, s'il vous plait.")
+ check("Tuez l'illumination maintenant.")
+ }
+ }
+ </pre>
+ <ul>
+ <li>
+ On <code>line 9</code> the client for our model is created.
+ </li>
+ <li>
+ On <code>line 11</code> a special method <code>debugAsk</code> is called.
+ It allows to check the winning intent and its callback parameters without actually
+ calling the intent.
+ </li>
+ <li>
+ <code>Lines 13-34</code> define all the test input sentences that should all
+ trigger <code>ls</code> intent.
+ </li>
+ </ul>
+ <p>
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
+ </p>
+ <pre class="brush: scala, highlight: []">
+ $ sbt executeTests
+ </pre>
+ </section>
+ <section>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You've created light switch data model and tested it.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#new_project">New Project</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#testing">Testing</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
+
+
diff --git a/examples/light_switch_ru.html b/examples/light_switch_ru.html
new file mode 100644
index 0000000..1fb5aa8
--- /dev/null
+++ b/examples/light_switch_ru.html
@@ -0,0 +1,485 @@
+---
+active_crumb: Light Switch RU <code><sub>ex</sub></code>
+layout: documentation
+id: light_switch_ru
+fa_icon: fa-cube
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column example">
+ <section id="overview">
+ <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ This example provides a very simple Russian language implementation for NLI-powered light switch. You can say something like
+ "Выключи свет по всем доме" or "Включи свет в детской".
+ By modifying intent callbacks using, for example, HomeKit or Arduino-based controllers you can provide the
+ actual light switching.
+ </p>
+ <p>
+ <b>Complexity:</b> <span class="complexity-two-star"><i class="fas fa-square"></i> <i class="fas fa-square"></i> <i class="far fa-square"></i></span><br/>
+ <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/lightswitch_ru">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
+ <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
+ </p>
+ </section>
+ <section id="new_project">
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
+ </p>
+ <pre class="brush: js, highlight: [8, 9, 10]">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft LightSwitch RU Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.apache.lucene" % "lucene-analyzers-common" % "8.11.2",
+ libraryDependencies += "org.languagetool" % "languagetool-core" % "5.8",
+ libraryDependencies += "org.languagetool" % "language-ru" % "5.8"
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
+ </pre>
+
+ <p>
+ On <code>lines 8, 9 and 10</code> added libraries, which used for support base NLP operations with Russian language.
+ </p>
+
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>lightswitch_model_ru.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>LightSwitchRuModel.scala</code> - Model implementation.</li>
+ <li><code>NCRuSemanticEntityParser.scala</code> - Semantic entity parser, custom implementation for Russian language.</li>
+ <li><code>NCRuLemmaPosTokenEnricher.scala</code> - Lemma and point of speech token enricher, custom implementation for Russian language.</li>
+ <li><code>NCRuStopWordsTokenEnricher.scala</code> - Stop-words token enricher, custom implementation for Russian language.</li>
+ <li><code>NCRuTokenParser.scala</code> - Token parser, custom implementation for Russian language.</li>
+ <li><code>LightSwitchRuModelSpec.scala</code> - Test that allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 10, 14, 17, 18, 20, 24]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | lightswitch_model_ru.yaml
+ | \--scala
+ | \--demo
+ | | LightSwitchRuModel.scala
+ | \--nlp
+ | +--entity
+ | | \--parser
+ | | NCRuSemanticEntityParser.scala
+ | \--token
+ | +--enricher
+ | | NCRuLemmaPosTokenEnricher.scala
+ | | NCRuStopWordsTokenEnricher.scala
+ | \--parser
+ | NCRuTokenParser.scala
+ \--test
+ \--scala
+ \--demo
+ LightSwitchRuModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ We are going to start with declaring the static part of our model using YAML which we will later load using
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>light_switch_ru.yaml</b></code>
+ file and replace its content with the following YAML:
+ </p>
+ <pre class="brush: js, highlight: [1, 8, 13, 21]">
+ macros:
+ "<TURN_ON>" : "{включить|включать|врубить|врубать|запустить|запускать|зажигать|зажечь}"
+ "<TURN_OFF>" : "{погасить|загасить|гасить|выключить|выключать|вырубить|вырубать|отключить|отключать|убрать|убирать|приглушить|приглушать|стоп}"
+ "<ENTIRE_OPT>" : "{весь|все|всё|повсюду|вокруг|полностью|везде|_}"
+ "<LIGHT_OPT>" : "{это|лампа|бра|люстра|светильник|лампочка|лампа|освещение|свет|электричество|электрика|_}"
+
+ elements:
+ - id: "ls:loc"
+ description: "Location of lights."
+ synonyms:
+ - "<ENTIRE_OPT> {здание|помещение|дом|кухня|детская|кабинет|гостиная|спальня|ванная|туалет|{большая|обеденная|ванная|детская|туалетная} комната}"
+
+ - id: "ls:on"
+ groups:
+ - "act"
+ description: "Light switch ON action."
+ synonyms:
+ - "<LIGHT_OPT> <ENTIRE_OPT> <TURN_ON>"
+ - "<TURN_ON> <ENTIRE_OPT> <LIGHT_OPT>"
+
+ - id: "ls:off"
+ groups:
+ - "act"
+ description: "Light switch OFF action."
+ synonyms:
+ - "<LIGHT_OPT> <ENTIRE_OPT> <TURN_OFF>"
+ - "<TURN_OFF> <ENTIRE_OPT> <LIGHT_OPT>"
+ - "без <ENTIRE_OPT> <LIGHT_OPT>"
+ </pre>
+ <p>There are number of important points here:</p>
+ <ul>
+ <li>
+ <code>Line 1</code> defines several macros that are used later on throughout the model's elements
+ to shorten the synonym declarations. Note how macros coupled with option groups
+ shorten overall synonym declarations 1000:1 vs. manually listing all possible word permutations.
+ </li>
+ <li>
+ <code>Lines 8, 13, 21</code> define three model elements: the location of the light, and actions to turn
+ the light on and off. Action elements belong to the same group <code>act</code> which
+ will be used in our intent, defined in <code>LightSwitchRuModel</code> class. Note that these model
+ elements are defined mostly through macros we have defined above.
+
+ </li>
+ </ul>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>LightSwitchRuModel</code> class as well.
+ </p>
+ </div>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Open <code>src/main/scala/demo/<b>LightSwitchRuModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [11, 12, 13, 20, 21, 24, 25, 32]">
+ package demo
+
+ import com.google.gson.Gson
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
+ import demo.nlp.entity.parser.NCRuSemanticEntityParser
+ import demo.lightswitch.nlp.token.enricher.*
+ import demo.lightswitch.nlp.token.parser.NCRuTokenParser
+ import scala.jdk.CollectionConverters.*
+
+ class LightSwitchRuModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.lightswitch.ru.ex", "LightSwitch Example Model RU", "1.0"),
+ new NCPipelineBuilder().
+ withTokenParser(new NCRuTokenParser()).
+ withTokenEnricher(new NCRuLemmaPosTokenEnricher()).
+ withTokenEnricher(new NCRuStopWordsTokenEnricher()).
+ withEntityParser(new NCRuSemanticEntityParser("lightswitch_model_ru.yaml")).
+ build
+ ):
+ @NCIntent("intent=ls term(act)={has(ent_groups, 'act')} term(loc)={# == 'ls:loc'}*")
+ def onMatch(
+ ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("act") actEnt: NCEntity,
+ @NCIntentTerm("loc") locEnts: List[NCEntity]
+ ): NCResult =
+ val action = if actEnt.getId == "ls:on" then "включить" else "выключить"
+ val locations = if locEnts.isEmpty then "весь дом" else locEnts.map(_.mkText).mkString(", ")
+
+ // Add HomeKit, Arduino or other integration here.
+ // By default - just return a descriptive action string.
+ NCResult(new Gson().toJson(Map("locations" -> locations, "action" -> action).asJava))
+ </pre>
+ <p>
+ The intent callback logic is very simple - we return a descriptive confirmation message
+ back (explaining what lights were changed). With action and location detected, you can add
+ the actual light switching using HomeKit or Arduino devices. Let's review this implementation step by step:
+ </p>
+ <ul>
+ <li>
+ On <code>line 11</code> our class extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
+ </li>
+ <li>
+ <code>Line 12</code> creates model configuration with most default parameters.
+ </li>
+ <li>
+ <code>Line 13</code> creates pipeline, based on custom Russian language components, which are described below.
+ <ul>
+ <li><code>NCRuTokenParser</code> - Token parser.</li>
+ <li><code>NCRuLemmaPosTokenEnricher</code> - Lemma and point of speech token enricher.</li>
+ <li><code>NCRuStopWordsTokenEnricher</code> - Stop-words token enricher.</li>
+ <li><code>NCRuSemanticEntityParser</code> - Semantic entity parser extending.</li>
+ </ul>
+ Note that <code>NCRuSemanticEntityParser</code> is based on semantic model definition
+ described in <code>lightswitch_model_ru.yaml</code> file.
+ </li>
+ <li>
+ <code>Lines 20 and 21</code> annotate intents <code>ls</code> and its callback method <code>onMatch()</code>.
+ Intent <code>ls</code> requires one action (a token belonging to the group <code>act</code>) and optional list of light locations
+ (zero or more tokens with ID <code>ls:loc</code>) - by default we assume the entire house as a default location.
+ </li>
+ <li>
+ <code>Lines 24 and 25</code> map terms from detected intent to the formal method parameters of the
+ <code>onMatch()</code> method.
+ </li>
+ <li>
+ On the <code>line 32</code> the intent callback simply returns a confirmation message.
+ </li>
+ </ul>
+
+ </section>
+ <section id="code">
+ <h2 class="section-title">Custom Components <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/parser/<b>NCRuTokenParser.scala</b></code> file and replace its content with the following code:
+ </p>
+
+ <pre class="brush: scala, highlight: [19]">
+ package demo.nlp.token.parser
+
+ import org.apache.nlpcraft.*
+ import org.languagetool.tokenizers.WordTokenizer
+ import scala.jdk.CollectionConverters.*
+
+ class NCRuTokenParser extends NCTokenParser:
+ private val tokenizer = new WordTokenizer
+
+ override def tokenize(text: String): List[NCToken] =
+ val toks = collection.mutable.ArrayBuffer.empty[NCToken]
+ var sumLen = 0
+
+ for ((word, idx) <- tokenizer.tokenize(text).asScala.zipWithIndex)
+ val start = sumLen
+ val end = sumLen + word.length
+
+ if word.strip.nonEmpty then
+ toks += new NCPropertyMapAdapter with NCToken:
+ override def getText: String = word
+ override def getIndex: Int = idx
+ override def getStartCharIndex: Int = start
+ override def getEndCharIndex: Int = end
+
+ sumLen = end
+
+ toks.toList
+ </pre>
+
+ <ul>
+ <li>
+ <code>NCRuTokenParser</code> is a simple wrapper which implements <code>NCTokenParser</code> based on
+ open source <a href="https://languagetool.org">Language Tool</a> library.
+ </li>
+ <li>
+ On <code>line 19</code> the <code>NCToken</code> instances created.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/enricher/<b>NCRuLemmaPosTokenEnricher.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [27, 28]">
+ package demo.nlp.token.enricher
+
+ import org.apache.nlpcraft.*
+ import org.languagetool.AnalyzedToken
+ import org.languagetool.tagging.ru.RussianTagger
+ import scala.jdk.CollectionConverters.*
+
+ class NCRuLemmaPosTokenEnricher extends NCTokenEnricher:
+ private def nvl(v: String, dflt : => String): String = if v != null then v else dflt
+
+ override def enrich(req: NCRequest, cfg: NCModelConfig, toks: List[NCToken]): Unit =
+ val tags = RussianTagger.INSTANCE.tag(toks.map(_.getText).asJava).asScala
+
+ require(toks.size == tags.size)
+
+ toks.zip(tags).foreach { case (tok, tag) =>
+ val readings = tag.getReadings.asScala
+
+ val (lemma, pos) = readings.size match
+ // No data. Lemma is word as is, POS is undefined.
+ case 0 => (tok.getText, "")
+ // Takes first. Other variants ignored.
+ case _ =>
+ val aTok: AnalyzedToken = readings.head
+ (nvl(aTok.getLemma, tok.getText), nvl(aTok.getPOSTag, ""))
+
+ tok.put("pos", pos)
+ tok.put("lemma", lemma)
+
+ () // Otherwise NPE.
+ }
+ </pre>
+
+ <ul>
+ <li>
+ <code>NCRuLemmaPosTokenEnricher</code> lemma and point of speech tokens enricher based on
+ open source <a href="https://languagetool.org">Language Tool</a> library.
+ </li>
+ <li>
+ On <code>line 27 and 28</code> the tokens are enriched by <code>pos</code> and <code>lemma</code> data.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/token/enricher/<b>NCRuStopWordsTokenEnricher.scala</b></code> file and replace its content with the following code:
+ </p>
+
+ <pre class="brush: scala, highlight: [17]">
+ package demo.nlp.token.enricher
+
+ import org.apache.lucene.analysis.ru.RussianAnalyzer
+ import org.apache.nlpcraft.*
+
+ class NCRuStopWordsTokenEnricher extends NCTokenEnricher:
+ private val stops = RussianAnalyzer.getDefaultStopSet
+
+ private def getPos(t: NCToken): String = t.get("pos").getOrElse(throw new NCException("POS not found in token."))
+ private def getLemma(t: NCToken): String = t.get("lemma").getOrElse(throw new NCException("Lemma not found in token."))
+
+ override def enrich(req: NCRequest, cfg: NCModelConfig, toks: List[NCToken]): Unit =
+ for (t <- toks)
+ val lemma = getLemma(t)
+ lazy val pos = getPos(t)
+
+ t.put(
+ "stopword",
+ lemma.length == 1 && !Character.isLetter(lemma.head) && !Character.isDigit(lemma.head) ||
+ stops.contains(lemma.toLowerCase) ||
+ pos.startsWith("PARTICLE") ||
+ pos.startsWith("INTERJECTION") ||
+ pos.startsWith("PREP")
+ )
+ </pre>
+
+ <ul>
+ <li>
+ <code>NCRuStopWordsTokenEnricher</code> is a stop-words tokens enricher based on
+ open source <a href="https://lucene.apache.org/">Apache Lucene</a> library.
+ </li>
+ <li>
+ On <code>line 17</code> tokens are enriched by <code>stopword</code> flags data.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/nlp/entity/parser/<b>NCRuSemanticEntityParser.scala</b></code> file and replace its content with the following code:
+ </p>
+
+ <pre class="brush: scala, highlight: [8, 12]">
+ package demo.nlp.entity.parser
+
+ import opennlp.tools.stemmer.snowball.SnowballStemmer
+ import demo.nlp.token.parser.NCRuTokenParser
+ import org.apache.nlpcraft.nlp.parsers.*
+
+ class NCRuSemanticEntityParser(src: String) extends NCSemanticEntityParser(
+ new NCSemanticStemmer:
+ private val stemmer = new SnowballStemmer(SnowballStemmer.ALGORITHM.RUSSIAN)
+ override def stem(txt: String): String = stemmer.synchronized { stemmer.stem(txt.toLowerCase).toString }
+ ,
+ new NCRuTokenParser(),
+ mdlSrcOpt = Option(src)
+ )
+ </pre>
+
+ <ul>
+ <li>
+ <code>NCRuSemanticEntityParser</code> extends <code>NCSemanticEntityParser</code>.
+ It uses stemmer implementation from <a href="https://opennlp.apache.org/">Apache OpenNLP</a> project.
+ </li>
+ </ul>
+ </section>
+
+ <section id="testing">
+ <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ The test defined in <code>LightSwitchRuModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intent <code>ls</code>:
+ </p>
+ <pre class="brush: scala, highlight: [9, 11]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class LightSwitchRuModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new LightSwitchRuModel)) { client =>
+ def check(txt: String): Unit =
+ require(client.debugAsk(txt, "userId", true).getIntentId == "ls")
+
+ check("Выключи свет по всем доме")
+ check("Выруби электричество!")
+ check("Включи свет в детской")
+ check("Включай повсюду освещение")
+ check("Включайте лампы в детской комнате")
+ check("Свет на кухне, пожалуйста, приглуши")
+ check("Нельзя ли повсюду выключить свет?")
+ check("Пожалуйста без света")
+ check("Отключи электричество в ванной")
+ check("Выключи, пожалуйста, тут всюду свет")
+ check("Выключай все!")
+ check("Свет пожалуйста везде включи")
+ check("Зажги лампу на кухне")
+ }
+ }
+ </pre>
+ <ul>
+ <li>
+ On <code>line 9</code> the client for our model is created.
+ </li>
+ <li>
+ On <code>line 11</code> a special method <code>debugAsk</code> is called.
+ It allows to check the winning intent and its callback parameters without actually
+ calling the intent.
+ </li>
+ <li>
+ <code>Lines 13-25</code> define all the test input sentences that should all
+ trigger <code>ls</code> intent.
+ </li>
+ </ul>
+ <p>
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
+ </p>
+ <pre class="brush: scala, highlight: []">
+ $ sbt executeTests
+ </pre>
+ </section>
+ <section>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You've created light switch data model and tested it.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#new_project">New Project</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#testing">Testing</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
+
+
diff --git a/examples/pizzeria.html b/examples/pizzeria.html
new file mode 100644
index 0000000..c1f95ee
--- /dev/null
+++ b/examples/pizzeria.html
@@ -0,0 +1,896 @@
+---
+active_crumb: Pizzeria <code><sub>ex</sub></code>
+layout: documentation
+id: pizzeria
+fa_icon: fa-cube
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column example">
+ <section id="overview">
+ <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ This example provides a simple pizzeria ordering system.
+ It demonstrates how to work with <code>ASK_DIALOG</code> states in callbacks and
+ how to process the systems which require confirmation logic.
+ </p>
+ <p>
+ <b>Complexity:</b> <span class="complexity-three-star"><i class="fas fa-gem"></i> <i class="fas fa-gem"></i> <i class="fas fa-gem"></i></span><br/>
+ <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/caclulator">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
+ <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
+ </p>
+ </section>
+ <section id="new_project">
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
+ </p>
+ <pre class="brush: js, highlight: []">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft Calculator Example",
+ version := "{{site.latest_version}}",
+ libraryDependencies += "org.apache.nlpcraft" % "nlpcraft" % "{{site.latest_version}}",
+ libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+ )
+ </pre>
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>pizzeria_model.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>PizzeriaModel.scala</code> - Scala class, model implementation.</li>
+ <li><code>PizzeriaOrder.scala</code> - Scala class, pizzeria order state representation.</li>
+ <li><code>PizzeriaModelPipeline.scala</code> - Scala class, model pipeline.</li>
+ <li><code>PizzeriaOrderMapper.scala</code> - Scala class, <code>NCEntityMapper</code> custom implementation.</li>
+ <li><code>PizzeriaOrderValidator.scala</code> - Scala class, <code>NCEntityValidator</code> custom implementation.</li>
+ <li><code>PizzeriaModelSpec.scala</code> - Scala tests class, which allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 11, 12, 13, 14, 15, 19]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | pizzeria_model.yaml
+ | \--scala
+ | \--demo
+ | \--components
+ | PizzeriaModelPipeline.scala
+ | PizzeriaOrderMapper.scala
+ | PizzeriaOrderValidator.scala
+ | PizzeriaModel.scala
+ | PizzeriaOrder.scala
+ \--test
+ \--scala
+ \--demo
+ PizzeriaModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ We are going to start with declaring the static part of our model using YAML which we will later load using
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>pizzeria_model.yaml</b></code>
+ file and replace its content with the following YAML:
+ </p>
+ <pre class="brush: js, highlight: [2, 9, 16, 23, 29, 35, 40, 46, 51]">
+ elements:
+ - id: "ord:pizza"
+ description: "Kinds of pizza."
+ values:
+ "margherita": [ ]
+ "carbonara": [ ]
+ "marinara": [ ]
+
+ - id: "ord:pizza:size"
+ description: "Size of pizza."
+ values:
+ "small": [ "{small|smallest|min|minimal|tiny} {size|piece|_}" ]
+ "medium": [ "{medium|intermediate|normal|regular} {size|piece|_}" ]
+ "large": [ "{big|biggest|large|max|maximum|huge|enormous} {size|piece|_}" ]
+
+ - id: "ord:drink"
+ description: "Kinds of drinks."
+ values:
+ "tea": [ ]
+ "coffee": [ ]
+ "cola": [ "{pepsi|sprite|dr. pepper|dr pepper|fanta|soda|cola|coca cola|cocacola|coca-cola}" ]
+
+ - id: "ord:yes"
+ description: "Confirmation (yes)."
+ synonyms:
+ - "{yes|yeah|right|fine|nice|excellent|good|correct|sure|ok|exact|exactly|agree}"
+ - "{you are|_} {correct|right}"
+
+ - id: "ord:no"
+ description: "Confirmation (no)."
+ synonyms:
+ - "{no|nope|incorrect|wrong}"
+ - "{you are|_} {not|are not|aren't} {correct|right}"
+
+ - id: "ord:stop"
+ description: "Stop and cancel all."
+ synonyms:
+ - "{stop|cancel|clear|interrupt|quit|close} {it|all|everything|_}"
+
+ - id: "ord:status"
+ description: "Order status information."
+ synonyms:
+ - "{present|current|_} {order|_} {status|state|info|information}"
+ - "what {already|_} ordered"
+
+ - id: "ord:finish"
+ description: "The order is over."
+ synonyms:
+ - "{i|everything|order|_} {be|_} {finish|ready|done|over|confirmed}"
+
+ - id: "ord:menu"
+ description: "Order menu."
+ synonyms:
+ - "{menu|carte|card}"
+ - "{products|goods|food|item|_} list"
+ - "{hi|help|hallo}"
+ </pre>
+ <p>There are number of important points here:</p>
+ <ul>
+ <li>
+ <code>Lines 1, 9, 16</code> define order elements, which present parts of orders.
+ </li>
+ <li>
+ <code>Lines 35, 40, 46, 51</code> define command elements, which are used to control order state.
+ </li>
+ <li>
+ <code>Lines 23, 29</code> define confirmation elements, which are used for commands confirmations or canceling.
+ </li>
+ </ul>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>PizzeriaModel</code> class as well.
+ </p>
+ </div>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Open <code>src/main/scala/demo/<b>PizzeriaModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [12, 27, 114, 115, 116, 126, 127, 135, 136, 143, 145, 147, 148, 158, 159, 168, 170, 173, 174, 187, 188, 201]">
+ package demo
+
+ import com.typesafe.scalalogging.LazyLogging
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.NCResultType.*
+ import org.apache.nlpcraft.annotations.*
+ import demo.{PizzeriaOrder as Order, PizzeriaOrderState as State}
+ import demo.PizzeriaOrderState.*
+ import demo.components.PizzeriaModelPipeline
+ import org.apache.nlpcraft.nlp.*
+
+ object PizzeriaExtractors:
+ def extractPizzaSize(e: NCEntity): String = e[String]("ord:pizza:size:value")
+ def extractQty(e: NCEntity, qty: String): Option[Int] =
+ Option.when(e.contains(qty))(e[String](qty).toDouble.toInt)
+ def extractPizza(e: NCEntity): Pizza =
+ Pizza(
+ e[String]("ord:pizza:value"),
+ e.get[String]("ord:pizza:size"),
+ extractQty(e, "ord:pizza:qty")
+ )
+ def extractDrink(e: NCEntity): Drink =
+ Drink(e[String]("ord:drink:value"), extractQty(e, "ord:drink:qty"))
+
+ import PizzeriaExtractors.*
+
+ object PizzeriaModel extends LazyLogging:
+ type Result = (NCResult, State)
+ private val UNEXPECTED_REQUEST =
+ new NCRejection("Unexpected request for current dialog context.")
+
+ private def getCurrentOrder()(using ctx: NCContext): Order =
+ val sess = ctx.getConversation.getData
+ val usrId = ctx.getRequest.getUserId
+ sess.get[Order](usrId) match
+ case Some(ord) => ord
+ case None =>
+ val ord = new Order()
+ sess.put(usrId, ord)
+ ord
+
+ private def mkResult(msg: String): NCResult = NCResult(msg, ASK_RESULT)
+ private def mkDialog(msg: String): NCResult = NCResult(msg, ASK_DIALOG)
+
+ private def doRequest(body: Order => Result)(using ctx: NCContext, im: NCIntentMatch): NCResult =
+ val o = getCurrentOrder()
+
+ logger.info(s"Intent '${im.getIntentId}' activated for text: '${ctx.getRequest.getText}'.")
+ logger.info(s"Before call [desc=${o.getState.toString}, resState: $o.")
+
+ val (res, resState) = body.apply(o)
+ o.setState(resState)
+
+ logger.info(s"After call [desc=$o, resState: $resState.")
+
+ res
+
+ private def askIsReady(): Result = mkDialog("Is order ready?") -> DIALOG_IS_READY
+
+ private def askSpecify(o: Order): Result =
+ require(!o.isValid)
+
+ o.findPizzaWithoutSize match
+ case Some(p) =>
+ mkDialog(s"Choose size (large, medium or small) for: '${p.name}'") -> DIALOG_SPECIFY
+ case None =>
+ require(o.isEmpty)
+ mkDialog("Please order something. Ask `menu` to look what you can order.") ->
+ DIALOG_SPECIFY
+
+ private def askShouldStop(): Result =
+ mkDialog("Should current order be canceled?") ->
+ DIALOG_SHOULD_CANCEL
+
+ private def doShowMenuResult(): NCResult =
+ mkResult(
+ "There are accessible for order: margherita, carbonara and marinara. " +
+ "Sizes: large, medium or small. " +
+ "Also there are tea, coffee and cola."
+ )
+
+ private def doShowMenu(state: State): Result = doShowMenuResult() -> state
+
+ private def doShowStatus(o: Order, state: State): Result =
+ mkResult(s"Current order state: $o.") -> state
+
+ private def askConfirm(o: Order): Result =
+ require(o.isValid)
+ mkDialog(s"Let's specify your order: $o. Is it correct?") -> DIALOG_CONFIRM
+
+ private def doResultWithClear(msg: String)(using ctx: NCContext, im: NCIntentMatch): Result =
+ val conv = ctx.getConversation
+ conv.getData.remove(ctx.getRequest.getUserId)
+ conv.clearStm(_ => true)
+ conv.clearDialog(_ => true)
+ mkResult(msg) -> DIALOG_EMPTY
+
+ private def doStop(o: Order)(using ctx: NCContext, im: NCIntentMatch): Result =
+ doResultWithClear(
+ if !o.isEmpty then "Everything cancelled. Ask `menu` to look what you can order."
+ else "Nothing to cancel. Ask `menu` to look what you can order."
+ )
+
+ private def doContinue(): Result = mkResult("OK, please continue.") -> DIALOG_EMPTY
+ private def askConfirmOrAskSpecify(o: Order): Result =
+ if o.isValid then askConfirm(o) else askSpecify(o)
+ private def askIsReadyOrAskSpecify(o: Order): Result =
+ if o.isValid then askIsReady() else askSpecify(o)
+ private def askStopOrDoStop(o: Order)(using ctx: NCContext, im: NCIntentMatch): Result =
+ if o.isValid then askShouldStop() else doStop(o)
+
+ import org.apache.nlpcraft.examples.pizzeria.PizzeriaModel.*
+
+ class PizzeriaModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.pizzeria.ex", "Pizzeria Example Model", "1.0"),
+ PizzeriaModelPipeline.PIPELINE
+ ) with LazyLogging:
+ // This method is defined in class scope and has package access level for tests reasons.
+ private[pizzeria] def doExecute(o: Order)(using ctx: NCContext, im: NCIntentMatch): Result =
+ require(o.isValid)
+ doResultWithClear(s"Executed: $o.")
+
+ private def doExecuteOrAskSpecify(o: Order)(using ctx: NCContext, im: NCIntentMatch): Result =
+ if o.isValid then doExecute(o) else askSpecify(o)
+
+ @NCIntent("intent=yes term(yes)={# == 'ord:yes'}")
+ def onYes(using ctx: NCContext, im: NCIntentMatch): NCResult = doRequest(
+ o => o.getState match
+ case DIALOG_CONFIRM => doExecute(o)
+ case DIALOG_SHOULD_CANCEL => doStop(o)
+ case DIALOG_IS_READY => askConfirmOrAskSpecify(o)
+ case DIALOG_SPECIFY | DIALOG_EMPTY => throw UNEXPECTED_REQUEST
+ )
+
+ @NCIntent("intent=no term(no)={# == 'ord:no'}")
+ def onNo(using ctx: NCContext, im: NCIntentMatch): NCResult = doRequest(
+ o => o.getState match
+ case DIALOG_CONFIRM | DIALOG_IS_READY => doContinue()
+ case DIALOG_SHOULD_CANCEL => askConfirmOrAskSpecify(o)
+ case DIALOG_SPECIFY | DIALOG_EMPTY => throw UNEXPECTED_REQUEST
+ )
+
+ @NCIntent("intent=stop term(stop)={# == 'ord:stop'}")
+ // It doesn't depend on order validity and dialog state.
+ def onStop(using ctx: NCContext, im: NCIntentMatch): NCResult = doRequest(askStopOrDoStop)
+
+ @NCIntent("intent=status term(status)={# == 'ord:status'}")
+ def onStatus(using ctx: NCContext, im: NCIntentMatch): NCResult = doRequest(
+ o => o.getState match
+ // Ignore `status`, confirm again.
+ case DIALOG_CONFIRM => askConfirm(o)
+ // Changes state.
+ case DIALOG_SHOULD_CANCEL => doShowStatus(o, DIALOG_EMPTY)
+ // Keeps same state.
+ case DIALOG_EMPTY | DIALOG_IS_READY | DIALOG_SPECIFY => doShowStatus(o, o.getState)
+ )
+
+ @NCIntent("intent=finish term(finish)={# == 'ord:finish'}")
+ def onFinish(using ctx: NCContext, im: NCIntentMatch): NCResult = doRequest(
+ o => o.getState match
+ // Like YES if valid.
+ case DIALOG_CONFIRM => doExecuteOrAskSpecify(o)
+ // Ignore `finish`, specify again.
+ case DIALOG_SPECIFY => askSpecify(o)
+ case DIALOG_EMPTY | DIALOG_IS_READY | DIALOG_SHOULD_CANCEL => askConfirmOrAskSpecify(o)
+ )
+
+ @NCIntent("intent=menu term(menu)={# == 'ord:menu'}")
+ // It doesn't depend and doesn't influence on order validity and dialog state.
+ def onMenu(using ctx: NCContext, im: NCIntentMatch): NCResult =
+ doRequest(o => doShowMenu(o.getState))
+
+ @NCIntent("intent=order term(ps)={# == 'ord:pizza'}* term(ds)={# == 'ord:drink'}*")
+ def onOrder(
+ using ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("ps") ps: List[NCEntity],
+ @NCIntentTerm("ds") ds: List[NCEntity]
+ ): NCResult = doRequest(
+ o =>
+ require(ps.nonEmpty || ds.nonEmpty);
+ // It doesn't depend on order validity and dialog state.
+ o.add(ps.map(extractPizza), ds.map(extractDrink));
+ askIsReadyOrAskSpecify(o)
+ )
+
+ @NCIntent("intent=orderSpecify term(size)={# == 'ord:pizza:size'}")
+ def onOrderSpecify(
+ using ctx: NCContext,
+ im: NCIntentMatch,
+ @NCIntentTerm("size") size: NCEntity
+ ): NCResult =
+ doRequest(
+ // If order in progress and has pizza with unknown size, it doesn't depend on dialog state.
+ o =>
+ if !o.isEmpty && o.fixPizzaWithoutSize(extractPizzaSize(size))
+ then askIsReadyOrAskSpecify(o)
+ else throw UNEXPECTED_REQUEST
+ )
+
+ override def onRejection(
+ using ctx: NCContext, im: Option[NCIntentMatch], e: NCRejection
+ ): Option[NCResult] =
+ if im.isEmpty || getCurrentOrder().isEmpty then throw e
+ Option(doShowMenuResult())
+ </pre>
+ <p>
+ There are few intents in the given model, which allow to prepare, change, confirm and cancel pizzeria orders.
+ Note please that given test model supports work with one single user.
+ Let's review this implementation step by step:
+ </p>
+ <ul>
+ <li>
+ On <code>line 12</code> declared <code>PizzeriaExtractors</code>, helper object, which provides
+ conversion methods from <code>NCEntity</code> objects and model data objects.
+ </li>
+ <li>
+ On <code>line 27</code> defined <code>PizzeriaModel</code> companion object, which contains
+ static content and helper methods.
+ </li>
+ <li>
+ On <code>line 114</code> our class <code>PizzeriaModel</code> extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
+ </li>
+ <li>
+ On <code>line 115</code> created model configuration with most default parameters.
+ </li>
+ <li>
+ On <code>line 116</code> represented pipeline, prepared in <code>PizzeriaModelPipeline</code> class.
+ </li>
+
+ <li>
+ <code>Lines 173 and 174</code> annotates intents <code>order</code> and its callback method <code>onOrder</code>.
+ Intent <code>order</code> requires lists of pizza and drinks in the order.
+ Note please, that at least one of these lists shouldn't be empty, otherwise intent is not triggered.
+ In the callback current order state is changed.
+ If order is in valid state, user receives order confirmation response "Is order ready?",
+ otherwise user receives response, which asks user to specify this order.
+ Both responses have type <code>ASK_DIALOG</code>.
+ </li>
+
+ <li>
+ Order pizza sizes can be specified by the model, as it was described above in <code>order</code> intent.
+ <code>Lines 187 and 188</code> annotates intents <code>orderSpecify</code> and its callback method <code>onOrderSpecify</code>.
+ Intent <code>orderSpecify</code> requires pizza size value parameter.
+ Callback checks that it was called just for suitable order state.
+ Current order state is changed and user receives order confirmation response "Is order ready?"
+ </li>
+ <li>
+ <code>Lines 126, 127 and 135, 136</code> annotates intents <code>yes</code> and <code>no</code>
+ with related callbacks <code>onYes</code> and <code>onNo</code>.
+ These intents are expected after user received confirmation responses with type <code>ASK_DIALOG</code>,
+ like "Is order ready?". Callbacks change order state or send some another confirmation requests to user,
+ depends on current order state.
+ </li>
+ <li>
+ <code>Lines 143 and 145, 147 and 148, 158 and 159, 168 and 170</code> annotates intents
+ <code>stop</code>, <code>status</code>, <code>finish</code> and <code>menu</code> intents
+ with related callbacks. They are order management commands, these actions are depends on current order state.
+ </li>
+ <li>
+ <code>line 201</code> annotates <code>onRejection</code> method,
+ which is called if there aren't triggered intents.
+ <code>stop</code>, <code>status</code>, <code>finish</code> and <code>menu</code> intents
+ with related callbacks. They are order management commands, these actions are depends on current order state.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/components/<b>PizzeriaOrderValidator.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: []">
+ package demo.components
+
+ import org.apache.nlpcraft.*
+
+ class PizzeriaOrderValidator extends NCEntityValidator:
+ override def validate(req: NCRequest, cfg: NCModelConfig, ents: List[NCEntity]): Unit =
+ def count(id: String): Int = ents.count(_.getId == id)
+
+ val cntPizza = count("ord:pizza")
+ val cntDrink = count("ord:drink")
+ val cntNums = count("stanford:number")
+ val cntSize = count("ord:pizza:size")
+
+ // Single size - it is order specification request.
+ if cntSize != 1 && cntSize > cntPizza then
+ throw new NCRejection("There are unrecognized pizza sizes in the request, maybe because some misprints.")
+
+ if cntNums > cntPizza + cntDrink then
+ throw new NCRejection("There are many unrecognized numerics in the request, maybe because some misprints.")
+ </pre>
+
+ <p>
+ <code>PizzeriaOrderValidator</code> is implementation of <code>NCEntityValidator</code>.
+ It is designed for validation order content and allows right away to reject invalid orders.
+ <p>
+
+ <p>
+ Open <code>src/main/scala/demo/components/<b>PizzeriaOrderMapper.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [11, 25, 30, 61]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import com.typesafe.scalalogging.LazyLogging
+ import org.apache.nlpcraft.NCResultType.ASK_DIALOG
+ import scala.collection.*
+
+ case class PizzeriaOrderMapperDesc(elementId: String, propertyName: String)
+
+ object PizzeriaOrderMapper:
+ extension(entity: NCEntity)
+ def position: Double =
+ val toks = entity.getTokens
+ (toks.head.getIndex + toks.last.getIndex) / 2.0
+ def tokens: List[NCToken] = entity.getTokens
+
+ private def str(es: Iterable[NCEntity]): String =
+ es.map(e => s"id=${e.getId}(${e.tokens.map(_.getIndex).mkString("[", ",", "]")})").
+ mkString("{", ", ", "}")
+
+ def apply(extra: PizzeriaOrderMapperDesc, dests: PizzeriaOrderMapperDesc*): PizzeriaOrderMapper =
+ new PizzeriaOrderMapper(extra, dests)
+
+ import PizzeriaOrderMapper.*
+
+ case class PizzeriaOrderMapper(
+ extra: PizzeriaOrderMapperDesc,
+ dests: Seq[PizzeriaOrderMapperDesc]
+ ) extends NCEntityMapper with LazyLogging:
+ override def map(req: NCRequest, cfg: NCModelConfig, ents: List[NCEntity]): List[NCEntity] =
+ def map(destEnt: NCEntity, destProp: String, extraEnt: NCEntity): NCEntity =
+ new NCPropertyMapAdapter with NCEntity:
+ destEnt.keysSet.foreach(k => put(k, destEnt(k)))
+ put[String](destProp, extraEnt[String](extra.propertyName).toLowerCase)
+ override val getTokens: List[NCToken] =
+ (destEnt.tokens ++ extraEnt.tokens).sortBy(_.getIndex)
+ override val getRequestId: String = req.getRequestId
+ override val getId: String = destEnt.getId
+
+ val destsMap = dests.map(p => p.elementId -> p).toMap
+ val destEnts = mutable.HashSet.empty ++ ents.filter(e => destsMap.contains(e.getId))
+ val extraEnts = ents.filter(_.getId == extra.elementId)
+
+ if destEnts.nonEmpty && extraEnts.nonEmpty && destEnts.size >= extraEnts.size then
+ val used = (destEnts ++ extraEnts).toSet
+ val dest2Extra = mutable.HashMap.empty[NCEntity, NCEntity]
+
+ for (extraEnt <- extraEnts)
+ val destEnt = destEnts.minBy(m => Math.abs(m.position - extraEnt.position))
+ destEnts -= destEnt
+ dest2Extra += destEnt -> extraEnt
+
+ val unrelated = ents.filter(e => !used.contains(e))
+ val artificial = for ((m, e) <- dest2Extra) yield map(m, destsMap(m.getId).propertyName, e)
+ val unused = destEnts
+
+ val res = (unrelated ++ artificial ++ unused).sortBy(_.tokens.head.getIndex)
+
+ logger.debug(s"Elements mapped [input=${str(ents)}, output=${str(res)}]")
+
+ res
+ else ents
+ </pre>
+
+ <p>
+ <code>PizzeriaOrderMapper</code> is implementation of <code>NCEntityMapper</code>.
+ It is designed for building complex compound entities based on another entities.
+ <p>
+ <ul>
+ <li>
+ On <code>line 11</code> declared <code>PizzeriaOrderMapper</code>, model companion object, which contains
+ helper methods.
+ </li>
+ <li>
+ On <code>line 25</code> declared <code>PizzeriaOrderMapper</code> model which implements <code>NCEntityMapper</code>.
+ </li>
+ <li>
+ On <code>line 30</code> defined helper method <code>map</code>, which clones <code>destEn</code> entity,
+ extend it by <code>extraEnt</code> tokens and <code>destProp</code> property and returns new entities
+ instead of passed inti the method.
+ </li>
+ <li>
+ <code>Line 61</code> defines <code>PizzeriaOrderMapper</code> result entities,
+ which will be processed further instead of passed into this component method.
+ </li>
+ </ul>
+
+ <p>
+ Open <code>src/main/scala/demo/components/<b>PizzeriaModelPipeline.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [14, 31, 37, 43]">
+ package demo.components
+
+ import edu.stanford.nlp.pipeline.StanfordCoreNLP
+ import opennlp.tools.stemmer.PorterStemmer
+ import org.apache.nlpcraft.nlp.parsers.*
+ import org.apache.nlpcraft.nlp.entity.parser.stanford.NCStanfordNLPEntityParser
+ import org.apache.nlpcraft.nlp.token.parser.stanford.NCStanfordNLPTokenParser
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.nlp.enrichers.NCEnStopWordsTokenEnricher
+ import org.apache.nlpcraft.nlp.parsers.{NCSemanticEntityParser, NCSemanticStemmer}
+ import java.util.Properties
+
+ object PizzeriaModelPipeline:
+ val PIPELINE: NCPipeline =
+ val stanford =
+ val props = new Properties()
+ props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner")
+ new StanfordCoreNLP(props)
+ val tokParser = new NCStanfordNLPTokenParser(stanford)
+ val stemmer = new NCSemanticStemmer():
+ private val ps = new PorterStemmer
+ override def stem(txt: String): String = ps.synchronized { ps.stem(txt) }
+
+ import PizzeriaOrderMapperDesc as D
+
+ new NCPipelineBuilder().
+ withTokenParser(tokParser).
+ withTokenEnricher(new NCEnStopWordsTokenEnricher()).
+ withEntityParser(new NCStanfordNLPEntityParser(stanford, Set("number"))).
+ withEntityParser(NCSemanticEntityParser(stemmer, tokParser, "pizzeria_model.yaml")).
+ withEntityMapper(
+ PizzeriaOrderMapper(
+ extra = D("ord:pizza:size", "ord:pizza:size:value"),
+ dests = D("ord:pizza", "ord:pizza:size")
+ )
+ ).
+ withEntityMapper(
+ PizzeriaOrderMapper(
+ extra = D("stanford:number", "stanford:number:nne"),
+ dests = D("ord:pizza", "ord:pizza:qty"), D("ord:drink", "ord:drink:qty")
+ )
+ ).
+ withEntityValidator(new PizzeriaOrderValidator()).
+ build
+ </pre>
+ <p>
+ In <code>PizzeriaModelPipeline</code> prepares model pipeline.
+ <p>
+ <ul>
+ <li>
+ On <code>line 14</code> pipeline is defined.
+ </li>
+ <li>
+ On <code>line 30</code> declared <code>NCSemanticEntityParser</code>
+ based on YAM model definition <code>pizzeria_model.yaml</code>.
+ </li>
+ <li>
+ On <code>lines 31 and 37</code> defined entity mappers <code>PizzeriaOrderMapper</code>, which
+ map <code>ord:pizza</code> elements with theirs sizes from <code>ord:pizza:size</code> and
+ quantities from <code>stanford:number</code>.
+ </li>
+ <li>
+ <code>Line 43</code> defines <code>PizzeriaOrderValidator</code> class, described above.
+ </li>
+ </ul>
+
+ </section>
+
+ <section id="testing">
+ <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ The test defined in <code>CalculatorModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intents <code>calc</code> and <code>calcMem</code>:
+ </p>
+ <pre class="brush: scala, highlight: [14, 48, 61, 96]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.NCResultType.*
+ import demo.PizzeriaModel.Result
+ import demo.PizzeriaOrderState.*
+ import org.scalatest.BeforeAndAfter
+ import org.scalatest.funsuite.AnyFunSuite
+
+ import scala.language.implicitConversions
+ import scala.util.Using
+ import scala.collection.mutable
+
+ object PizzeriaModelSpec:
+ type Request = (String, NCResultType)
+ private class ModelTestWrapper extends PizzeriaModel:
+ private var o: PizzeriaOrder = _
+
+ override def doExecute(o: PizzeriaOrder)(using ctx: NCContext, im: NCIntentMatch): Result =
+ val res = super.doExecute(o)
+ this.o = o
+ res
+
+ def getLastExecutedOrder: PizzeriaOrder = o
+ def clearLastExecutedOrder(): Unit = o = null
+
+ private class Builder:
+ private val o = new PizzeriaOrder
+ o.setState(DIALOG_EMPTY)
+ def withPizza(name: String, size: String, qty: Int): Builder =
+ o.add(Seq(Pizza(name, Some(size), Some(qty))), Seq.empty)
+ this
+ def withDrink(name: String, qty: Int): Builder =
+ o.add(Seq.empty, Seq(Drink(name, Some(qty))))
+ this
+ def build: PizzeriaOrder = o
+
+ import PizzeriaModelSpec.*
+
+ class PizzeriaModelSpec extends AnyFunSuite with BeforeAndAfter:
+ private val mdl = new ModelTestWrapper()
+ private val client = new NCModelClient(mdl)
+ private val msgs = mutable.ArrayBuffer.empty[mutable.ArrayBuffer[String]]
+ private val errs = mutable.HashMap.empty[Int, Throwable]
+
+ private var testNum: Int = 0
+
+ after {
+ if client != null then client.close()
+
+ for ((seq, num) <- msgs.zipWithIndex)
+ println("#" * 150)
+ for (line <- seq) println(line)
+ errs.get(num) match
+ case Some(err) => err.printStackTrace()
+ case None => // No-op.
+
+ require(errs.isEmpty, s"There are ${errs.size} errors above.")
+ }
+
+ private def dialog(exp: PizzeriaOrder, reqs: Request*): Unit =
+ val testMsgs = mutable.ArrayBuffer.empty[String]
+ msgs += testMsgs
+
+ testMsgs += s"Test: $testNum"
+
+ for (((txt, expType), idx) <- reqs.zipWithIndex)
+ try
+ mdl.clearLastExecutedOrder()
+ val resp = client.ask(txt, "userId")
+
+ testMsgs += s">> Request: $txt"
+ testMsgs += s">> Response: '${resp.getType}': ${resp.getBody}"
+
+ if expType != resp.getType then
+ errs += testNum -> new Exception(s"Unexpected result type [num=$testNum, txt=$txt, expected=$expType, type=${resp.getType}]")
+
+ // Check execution result on last request.
+ if idx == reqs.size - 1 then
+ val lastOrder = mdl.getLastExecutedOrder
+ def s(o: PizzeriaOrder) = if o == null then null else s"Order [state=${o.getState}, desc=$o]"
+ val s1 = s(exp)
+ val s2 = s(lastOrder)
+ if s1 != s2 then
+ errs += testNum ->
+ new Exception(
+ s"Unexpected result [num=$testNum, txt=$txt]" +
+ s"\nExpected: $s1" +
+ s"\nReal : $s2"
+ )
+ catch
+ case e: Exception => errs += testNum -> new Exception(s"Error during test [num=$testNum]", e)
+
+ testNum += 1
+
+ test("test") {
+ given Conversion[String, Request] with
+ def apply(txt: String): Request = (txt, ASK_DIALOG)
+
+ dialog(
+ new Builder().withDrink("tea", 2).build,
+ "Two tea",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().
+ withPizza("carbonara", "large", 1).
+ withPizza("marinara", "small", 1).
+ withDrink("tea", 1).
+ build,
+ "I want to order carbonara, marinara and tea",
+ "large size please",
+ "smallest",
+ "yes",
+ "correct" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().withPizza("carbonara", "small", 2).build,
+ "carbonara two small",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().withPizza("carbonara", "small", 1).build,
+ "carbonara",
+ "small",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ null,
+ "marinara",
+ "stop" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().
+ withPizza("carbonara", "small", 2).
+ withPizza("marinara", "large", 4).
+ withDrink("cola", 3).
+ withDrink("tea", 1).
+ build,
+ "3 cola",
+ "one tea",
+ "carbonara 2",
+ "small",
+ "4 marinara big size",
+ "menu" -> ASK_RESULT,
+ "done",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().
+ withPizza("margherita", "small", 2).
+ withPizza("marinara", "small", 1).
+ withDrink("tea", 3).
+ build,
+ "margherita two, marinara and three tea",
+ "small",
+ "small",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().
+ withPizza("margherita", "small", 2).
+ withPizza("marinara", "large", 1).
+ withDrink("cola", 3).
+ build,
+ "small margherita two, marinara big one and three cola",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+
+ dialog(
+ new Builder().
+ withPizza("margherita", "small", 1).
+ withPizza("marinara", "large", 2).
+ withDrink("coffee", 2).
+ build,
+ "small margherita, 2 marinara and 2 coffee",
+ "large",
+ "yes",
+ "yes" -> ASK_RESULT
+ )
+ }
+ </pre>
+
+ <p>
+ <code>PizzeriaModelSpec</code> is complex test, which is designed as dialog with Pizzeria bot.
+ </p>
+
+ <ul>
+ <li>
+ On <code>line 14</code> declared <code>PizzeriaModelSpec</code>, test companion object, which contains
+ static content and helper methods.
+ </li>
+ <li>
+ On <code>line 48</code> defined <code>after</code> block.
+ It closes model client and prints test results.
+ </li>
+ <li>
+ On <code>line 61</code> defined test helper method <code>dialog</code>.
+ It sends request to model via <code>ask</code> method and accumulates execution results.
+ </li>
+ <li>
+ On <code>line 96</code> defined main test block.
+ It contains user request descriptions and expected results on them, taking into account order state.
+ </li>
+ </ul>
+ <p>
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
+ </p>
+ <pre class="brush: scala, highlight: []">
+ PS C:\apache\incubator-nlpcraft-examples\pizzeria> sbt executeTests
+ </pre>
+ </section>
+ <section>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You've created pizzeria model and tested it.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#new_project">New Project</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#testing">Testing</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
+
+
diff --git a/examples/sql_model.html b/examples/sql_model.html
deleted file mode 100644
index 90eb96d..0000000
--- a/examples/sql_model.html
+++ /dev/null
@@ -1,300 +0,0 @@
----
-active_crumb: SQL Model <code><sub>ex</sub></code>
-layout: documentation
-id: sql_model
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This example demonstrates a relatively complete implementation of natural language interface to a
- SQL database. This is a non-trivial example, spanning over a thousand lines of code, and it can
- act as a basis for production-ready implementation.
- </p>
- <p>
- Note that a significant part of the implementation is not directly related to NLPCraft but rather deals
- with SQL statement construction. We specifically decided not to use any 3rd party libraries for it to
- show what can be done "from scratch".
- </p>
- <p>
- <b>Complexity:</b> <span class="complexity-three-star"><i class="fas fa-gem"></i> <i class="fas fa-gem"></i> <i class="fas fa-gem"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="background">
- <h2 class="section-title">Background <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many of modern existing natural language-to-SQL implementations use variations of
- deep learning approach where you first train the neural network on a pre-created training set and
- then get to use the trained network to provide (infer) probabilistic answers for the new input sentences. Although the latest
- natural language-to-SQL attempts to implement this approach are getting into 90% percentile of accuracy - they remain largely
- unusable for the vast swath of enterprise applications where non-deterministic nature of such systems
- renders them practically useless. The apparent problem with non-deterministic systems like that is that the user
- never knows whether a given answer is correct or incorrect. In these use cases users can't tolerate the fact that
- in 5-10% of the cases the answer will be invalid. Try that for your HR or revenue reporting system,
- prescriptive analytics systems, and so on...
- </p>
- <p>
- It's important to note, however, that for many non-mission-critical systems such non-determinism does
- not pose a significant problem. We happily accept such imprecision when asking for direction on our
- mobile devices, unlock our phones using face or fingerprint recognition, when performing sentiment
- analysis or trying to detect faces of our friends across thousands of photographs. Cost of retries,
- as well as the cost of initial errors, is insignificant in these cases. The same cost, however, in many business
- applications can be too significant to tolerate.
- </p>
- <p>
- As you may have learned by now, NLPCraft uses a <i>fully deterministic approach</i> in an attempt to
- match the user input against a defined set of intents. If it finds the matching intent - it guarantees that
- match for a given intent. If no matching intent can be found - it returns the negative
- result without any ambiguity. In other words, if the answer is given, it is deterministically guaranteed to be correct.
- Another positive side-effect of this approach is the fact that such matching logic is traceable, i.e.
- the user can see why given user input was matched against a certain intent (and not any other). Such
- traceability of the comprehension logic (or explainability vs "black box" approach from deep
- learning techniques) is often critical for many real-life business applications.
- </p>
- <p>
- On the flip side, one of the downsides of such an approach in general is the need to have a detailed,
- domain-specific model
- for each SQL database (when dealing with SQL databases). Building such a
- model can be a non-trivial and time consuming experience. That is where NLPCraft brings a lot of built-in tooling
- to simplify this task dramatically.
- </p>
- <div class="bq info">
- <b>Source Code</b>
- <p>
- Due to size of this example the entire source code for it can be found on <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">GitHub</a>.
- </p>
- </div>
- </section>
- <section id="sql">
- <h2 class="section-title">Sample Database <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to be building natural language interface against slightly modified "Northwind" sample database
- from <a target="_" href="https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/linq/downloading-sample-databases#get-the-northwind-sample-database-for-sql-server">Microsoft SQL Server</a>:
- </p>
- <figure>
- <img class="img-fluid" src="/images/sql_example_model.png" alt="">
- <figcaption><b>Fig 1.</b> SQL Schema</figcaption>
- </figure>
- <p>
- You can find SQL script creating this database and populating it with the sample data at
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/northwind.sql" target="github"><code>db/northwind.sql</code></a>
- file in the root of the example.
- </p>
- </section>
- <section id="h2">
- <h2 class="section-title">H2 Database <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In our example we are using H2 database instance. For convenience, example provides a command line application
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/db/SqlServer.scala" target="github"><code>db/SqlServer.scala</code></a>
- that automatically starts local H2 database instance with default configuration (localhost on port 9092) and initializes it
- using <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/northwind.sql" target="github"><code>db/northwind.sql</code></a> script.
- </p>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Creating data model manually for the existing SQL schema can be a daunting task. NLPCraft provides the tool
- that can scan SQL schema and create working stub of such model automatically. All you have to do then is to
- add any necessary modifications to it.
- </p>
- <p>
- <a href="/tools/sql_model_gen.html">SQL Model Generation</a> is a Java-based utility that takes
- JDBC configuration, reads database schema using it and creates initial JSON or YAML stub for the
- data model. This stub then can be used as is or be extended further.
- </p>
- <p>
- Examples specifically comes with two pre-generated files:
- </p>
- <ul>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/sql_model_init.yaml" target="github"><code>sql_model_init.yaml</code></a> - the initial file that was generated straight out of
- SQL Model Generator (see above for the instructions).
- </li>
- </ul>
- </section>
- <section id="impl">
- <h2 class="section-title">Implementation <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Implementation mainly consists of these files:
- </p>
- <ul>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala" target="github"><code>SqlModel.scala</code></a> - the code behind the data model that loads YAML-defined model and
- defines all intents.
- </li>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/db/SqlBuilder.scala" target="github"><code>db/SqlBuilder.scala</code></a> - the main utility that takes object model provided by
- <a href="/tools/sql_model_gen.html">SQL Model Generation</a>
- and builds a SQL query to execute. Note that we elected to build this functionality
- from scratch to illustrate how it can be done. You are free, of course, to use many
- of the existing libraries to help achieve this goal.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Assuming that project is located in <code>~/SqlModel</code> folder - let's go to into this directory and
- run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/SqlModel
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component
- that you generally need to start it only once for this and any other examples.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/sql/target/classes --mdls=demo.SqlModel
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created SQL model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#background">Background</a></li>
- <li><a href="#sql">Sample Database</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#impl">Implementation</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/examples/time.html b/examples/time.html
new file mode 100644
index 0000000..a249c47
--- /dev/null
+++ b/examples/time.html
@@ -0,0 +1,326 @@
+---
+active_crumb: Time <code><sub>ex</sub></code>
+layout: documentation
+id: time
+fa_icon: fa-cube
+---
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div class="col-md-8 second-column example">
+ <section id="overview">
+ <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ This example provides a very simple implementation for world time bot. You can say something like
+ "What time is it now in New York City" or "What's the local time?".
+ </p>
+ <p>
+ <b>Complexity:</b> <span class="complexity-one-star"><i class="fas fa-star"></i> <i class="far fa-star"></i> <i class="far fa-star"></i></span><br/>
+ <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/time">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
+ <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
+ </p>
+ </section>
+ <section id="new_project">
+ <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You can create new Scala projects in many ways - we'll use SBT
+ to accomplish this task. Make sure that <code>build.sbt</code> file has the following content:
+ </p>
+ <pre class="brush: js, highlight: []">
+ ThisBuild / version := "0.1.0-SNAPSHOT"
+ ThisBuild / scalaVersion := "3.1.3"
+ lazy val root = (project in file("."))
+ .settings(
+ name := "NLPCraft Time Example",
+ version := "{{site.latest_version}}"
+ )
+ </pre>
+ <p><b>NOTE: </b>use the latest versions of Scala and ScalaTest.</p>
+ <p>Create the following files so that resulting project structure would look like the following:</p>
+ <ul>
+ <li><code>time_model.yaml</code> - YAML configuration file, which contains model description.</li>
+ <li><code>cities_timezones.txt</code> - Cities timezones database.</li>
+ <li><code>TimeModel.scala</code> - Scala class, model implementation.</li>
+ <li><code>CitiesDataProvider.scala</code> - Scala class, helper service which loads timezones database.</li>
+ <li><code>GeoManager.scala</code> - Scala class, helper service which provides cities timezones information for user request.</li>
+ <li><code>TimeModelSpec.scala</code> - Scala tests class, which allows to test your model.</li>
+ </ul>
+ <pre class="brush: plain, highlight: [7, 10, 14]">
+ | build.sbt
+ +--project
+ | build.properties
+ \--src
+ +--main
+ | +--resources
+ | | time_model.yaml
+ | | cities_timezones.txt
+ | \--scala
+ | \--demo
+ | \--utils
+ | \--cities
+ | CitiesDataProvider.scala
+ | \--keycdn
+ | GeoManager.scala
+ | TimeModel.scala
+ \--test
+ \--scala
+ \--demo
+ TimeModelSpec.scala
+ </pre>
+ </section>
+ <section id="model">
+ <h2 class="section-title">Data Model<a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ We are going to start with declaring the static part of our model using YAML which we will later load using
+ <code>NCModelAdapter</code> in our Scala-based model implementation.
+ Open <code>src/main/resources/<b>time.yaml</b></code>
+ file and replace its content with the following YAML:
+ </p>
+ <pre class="brush: js, highlight: [1, 10, 17, 25]">
+ macros:
+ "<OF>": "{of|for|per}"
+ "<CUR>": "{current|present|now|local}"
+ "<TIME>": "{time <OF> day|day time|date|time|moment|datetime|hour|o'clock|clock|date time|date and time|time and date}"
+ elements:
+ - id: "x:time"
+ description: "Date and/or time token indicator."
+ synonyms:
+ - "{<CUR>|_} <TIME>"
+ - "what <TIME> {is it now|now|is it|_}"
+ </pre>
+ <p>There are number of important points here:</p>
+ <ul>
+ <li>
+ <code>Line 1</code> defines several macros that are used later on throughout the model's elements
+ to shorten the synonym declarations. Note how macros coupled with option groups
+ shorten overall synonym declarations 1000:1 vs. manually listing all possible word permutations.
+ </li>
+ <li>
+ On <code>line 6</code> defined <code>x:time</code> model elements, which
+ will be used in our intent, defined in <code>TimeModel</code> class. Note that these model
+ elements are defined mostly through macros we have defined above.
+ </li>
+ </ul>
+ <div class="bq info">
+ <p><b>YAML vs. API</b></p>
+ <p>
+ As usual, this YAML-based static model definition is convenient but totally optional. All elements definitions
+ can be provided programmatically inside Scala model <code>TimeModel</code> class as well.
+ </p>
+ </div>
+ </section>
+ <section id="code">
+ <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ Open <code>src/main/scala/demo/<b>TimeModel.scala</b></code> file and replace its content with the following code:
+ </p>
+ <pre class="brush: scala, highlight: [16, 17, 18, 19, 20, 21, 56, 57, 70, 71]">
+ package demo
+
+ import com.fasterxml.jackson.core.JsonProcessingException
+ import com.fasterxml.jackson.databind.ObjectMapper
+ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+ import org.apache.nlpcraft.*
+ import org.apache.nlpcraft.annotations.*
+ import demo.utils.cities.*
+ import demo.utils.keycdn.GeoManager
+ import org.apache.nlpcraft.internal.util.NCResourceReader
+ import org.apache.nlpcraft.nlp.parsers.NCOpenNLPEntityParser
+ import java.time.format.DateTimeFormatter
+ import java.time.format.FormatStyle.MEDIUM
+ import java.time.*
+
+ @NCIntent("fragment=city term(city)~{# == 'opennlp:location'}")
+ @NCIntent("intent=intent2 term~{# == 'x:time'} fragment(city)")
+ @NCIntent("intent=intent1 term={# == 'x:time'}")
+ class TimeModel extends NCModelAdapter(
+ NCModelConfig("nlpcraft.time.ex", "Time Example Model", "1.0"),
+ new NCPipelineBuilder().
+ withSemantic("en", "time_model.yaml").
+ withEntityParser(NCOpenNLPEntityParser(
+ NCResourceReader.getPath("opennlp/en-ner-location.bin"))
+ ).
+ build
+ ):
+ private val FMT: DateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(MEDIUM)
+
+ private val citiesData: Map[City, CityData] = CitiesDataProvider.get
+
+ private def mkResult(
+ city: String, cntry: String, tmz: String, lat: Double, lon: Double
+ ): NCResult =
+ val m =
+ Map(
+ "city" -> capitalize(city),
+ "country" -> capitalize(cntry),
+ "timezone" -> tmz,
+ "lat" -> lat,
+ "lon" -> lon,
+ "localTime" -> ZonedDateTime.now(ZoneId.of(tmz)).format(FMT)
+ )
+
+ try
+ NCResult(new ObjectMapper(new YAMLFactory).writeValueAsString(m))
+ catch
+ case e: JsonProcessingException =>
+ throw new RuntimeException("YAML conversion error.", e)
+
+ private def capitalize(s: String): String =
+ if s == null || s.isEmpty
+ then s
+ else s"${s.substring(0, 1).toUpperCase}${s.substring(1, s.length)}"
+
+ @NCIntentRef("intent2")
+ private def onRemoteMatch(
+ ctx: NCContext, im: NCIntentMatch, @NCIntentTerm("city") cityEnt: NCEntity
+ ): NCResult =
+ val cityName: String = cityEnt.mkText
+
+ val (city, data) =
+ citiesData.find(_._1.name.equalsIgnoreCase(cityName)).
+ getOrElse(
+ throw new NCRejection(String.format("No timezone mapping for %s.", cityName))
+ )
+
+ mkResult(city.name, city.country, data.timezone, data.latitude, data.longitude)
+
+ @NCIntentRef("intent1")
+ private def onLocalMatch(ctx: NCContext, im: NCIntentMatch): NCResult =
+ val geo = GeoManager.get(ctx.getRequest).getOrElse(GeoManager.getSiliconValley)
+
+ mkResult(geo.city, geo.country_name, geo.timezone, geo.latitude, geo.longitude)
+ </pre>
+ <p>
+ There are two intents, for local and remote location time. Result is represented as JSON value.
+ Let's review this implementation step by step:
+ </p>
+ <ul>
+ <li>
+ On <code>line 19</code> our class extends <code>NCModelAdapter</code> that allows us to pass
+ prepared configuration and pipeline into model.
+ </li>
+ <li>
+ On <code>line 16</code> created <code>IDL fragment</code>, which is used in <code>intent2</code> definition below.
+ </li>
+ <li>
+ On <code>lines 17 and 18</code> annotates two intents definitions <code>intent1</code> and <code>intent2</code> ,
+ callbacks below have references on them by their identifiers.
+ </li>
+ <li>
+ On <code>line 20</code> created model configuration with most default parameters.
+ </li>
+ <li>
+ On <code>line 21</code> created pipeline, based on built components.
+ <ul>
+ <li>This pipeline is based on built EN semantic entity enrichers, configured with <code>time_model.yaml</code>.</li>
+ <li>Also there is used entity parser <code>NCOpenNLPEntityParser</code>,
+ configured with <code>opennlp/en-ner-location.bin</code> for detection GEO locations.
+ </li>
+ </ul>
+ Look at documentations of these built components for more details.
+ </li>
+ <li>
+ <code>Lines 56 and 57</code> annotates intents <code>intent2</code> and its callback method <code>onRemoteMatch</code>.
+ This intent requires one mandatory entity - city, which is used for getting time for its timezone.
+ </li>
+ <li>
+ <code>Lines 70 and 71</code> annotates intents <code>intent1</code> and its callback method <code>onLocalMatch</code>.
+ This intent is triggered by default and tries to detect timezone by request data and return time for this timezone.
+ Otherwise, it returns Silicon Valley current time.
+ </li>
+ </ul>
+
+ <p>
+ Implementations of helper classes <code>GeoManager</code> and <code>CitiesDataProvider</code> are not related to given example.
+ Just copy these classes and <code>cities_timezones.txt</code> from project source code into your demo project.
+ </p>
+ </section>
+
+ <section id="testing">
+ <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ The test defined in <code>TimeModelSpec</code> allows to check that all input test sentences are
+ processed correctly and trigger the expected intents <code>intent2</code> and <code>intent1</code>:
+ </p>
+ <pre class="brush: scala, highlight: [9, 11]">
+ package demo
+
+ import org.apache.nlpcraft.*
+ import org.scalatest.funsuite.AnyFunSuite
+ import scala.util.Using
+
+ class TimeModelSpec extends AnyFunSuite:
+ test("test") {
+ Using.resource(new NCModelClient(new TimeModel())) { client =>
+ def check(txt: String, intentId: String): Unit =
+ require(client.debugAsk(txt, "userId", true).getIntentId == intentId)
+
+ check("What time is it now in New York City?", "intent2")
+ check("What's the current time in Moscow?", "intent2")
+ check("Show me time of the day in London.", "intent2")
+ check("Can you please give me the Tokyo's current date and time.", "intent2")
+
+ check("What's the local time?", "intent1")
+ }
+ }
+ </pre>
+ <ul>
+ <li>
+ On <code>line 9</code> the client for our model is created.
+ </li>
+ <li>
+ On <code>line 11</code> a special method <code>debugAsk</code> is called.
+ It allows to check the winning intent and its callback parameters without actually
+ calling the intent.
+ </li>
+ <li>
+ <code>Lines 13-18</code> define all the test input sentences that should all
+ trigger <code>ls</code> intent.
+ </li>
+ </ul>
+ <p>
+ You can run this test via SBT task <code>executeTests</code> or using IDE.
+ </p>
+ <pre class="brush: scala, highlight: []">
+ PS C:\apache\incubator-nlpcraft-examples\time> sbt executeTests
+ </pre>
+ </section>
+ <section>
+ <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
+ <p>
+ You've created time data model and tested it.
+ </p>
+ </section>
+</div>
+<div class="col-md-2 third-column">
+ <ul class="side-nav">
+ <li class="side-nav-title">On This Page</li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#new_project">New Project</a></li>
+ <li><a href="#model">Data Model</a></li>
+ <li><a href="#code">Model Class</a></li>
+ <li><a href="#testing">Testing</a></li>
+ {% include quick-links.html %}
+ </ul>
+</div>
+
+
+
+
+
+
diff --git a/examples/weather_bot.html b/examples/weather_bot.html
deleted file mode 100644
index f188701..0000000
--- a/examples/weather_bot.html
+++ /dev/null
@@ -1,582 +0,0 @@
----
-active_crumb: Weather Bot <code><sub>ex</sub></code>
-layout: documentation
-id: weather_bot
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This example demonstrates relatively complete NLI-based weather service with JSON output and a non-trivial
- intent matching logic. It uses <a target="new" href="https://openweathermap.org/api/one-call-api">OpenWeather</a>
- REST service for the actual weather information.
- </p>
- <div class="bq info">
- <p>
- <b>NOTE:</b> you must provide OpenWeather API key in <code>OWM_API_KEY</code> system property when running
- the data probe. See <a target=_ href="https://openweathermap.org/api">https://openweathermap.org/api</a> for more information.
- </p>
- </div>
- <p>
- <b>Complexity:</b> <span class="complexity-two-star"><i class="fas fa-square"></i> <i class="fas fa-square"></i> <i class="far fa-square"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/weather">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="new_project">
- <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can create new Java project in many ways - we'll use <a href="/tools/script.html">NLPCraft CLI</a>
- to accomplish this task:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-prj-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-prj-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-prj-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=Weather --outputDir=~ --pkgName=demo --mdlType=json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/Weather</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Java and Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-prj-out" role="tabpanel">
- <p></p>
- <img alt="" class="img-fluid" src="/images/weather_bot_fig1.png">
- </div>
- </div>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to start with declaring the static part of our model using JSON which we will later load using
- <code>NCModelFileAdapter</code> in our Java-based model implementation. Open <code>src/main/resources/<b>weather.json</b></code>
- and replace its content with the following JSON:
- </p>
- <pre class="brush: js, highlight: [10, 18, 28, 38]">
-{
- "id": "nlpcraft.weather.ex",
- "name": "Weather Example Model",
- "version": "1.0",
- "description": "Weather example model.",
- "macros": [
- ],
- "elements": [
- {
- "id": "wt:phen",
- "description": "Weather phenomenon.",
- "synonyms": [
- "{high sea|severe weather|hail|heat wave|cold wave|derecho|supercell|avalanche|cyclone|wildfire|landslide|firestorm|dust storm|thunder snow|winter storm|cloudburst|shower|condensation|precipitation|drizzle|rainstorm|rain storm|rainfall|rain|storm|sun|sunshine|cloud|hot|cold|dry|wet|wind|hurricane|typhoon|sand-storm|sand storm|tornado|humid|fog|snow|smog|black ice|haze|thundershower|thundersnow|sleet|drought|wildfire|blizzard|avalanche|mist|thunderstorm}",
- "{weather {condition|temp|temperature|data|_}|condition|temp|temperature}"
- ]
- },
- {
- "id": "wt:hist",
- "description": "History (past) indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{history|past|previous}"
- ]
- },
- {
- "id": "wt:curr",
- "description": "Current indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{current|today|now|right now}"
- ]
- },
- {
- "id": "wt:fcast",
- "description": "Forecast (future) indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{future|forecast|prognosis|prediction}"
- ]
- }
- ]
-}
- </pre>
- <p>There are number of important points here:</p>
- <ul>
- <li>
- <code>Line 10</code> defines an element <code>wt:phen</code> for various weather phenomenon.
- </li>
- <li>
- <code>Line 18</code> defines an element <code>wt:hist</code> whose presence will indicate the
- request for the past (history) weather information.
- </li>
- <li>
- <code>Line 28</code> defines an element <code>wt:curr</code> whose presence will indicate the
- request for the current (local) weather information.
- </li>
- <li>
- <code>Line 38</code> defines an element <code>wt:fcast</code> whose presence will indicate the
- request for the future (forecast) weather information.
- </li>
- </ul>
- <p>
- Now that our model is ready let's create a Java class that would load this model and define the intent
- that uses the model elements we have just defined.
- </p>
- <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Below is a full source code for our implementation. Note that this code uses several
- external tools for IP-based geo-location and the weather information
- provider (<a target=_ href="https://openweathermap.org/api">OpenWeather</a>). Note also that despite its apparent simplicity the model logic implementation is non-trivial.
- Significant portion of the code deals with a complex <b>temporal and geographical ambiguity</b>, i.e. the sentences like the these:
- </p>
- <dl>
- <dt>
- <code>'show my local weather'</code> vs <code>'show local Boston weather'</code>
- </dt>
- <dd>
- In the first sentence the word <code>local</code> indicates that the user is asking about her local weather
- (i.e. the weather for the location based on the IP address of the user's REST agent). However, in the
- second sentence the same word <code>local</code> is effectively ignored as user clearly indicated the city
- for the weather request (Boston).
- </dd>
- <dt>
- <code>'show weather forecast'</code> vs <code>'show weather forecast for today'</code> vs <code>'show last week weather forecast'</code>
- </dt>
- <dd>
- In these three sentences the word <code>forecast</code> defines future, current and past timeframe for the
- weather request. It acts as a future indicator in the first sentence and is effectively ignored in the second
- and third sentences.
- </dd>
- </dl>
- <div class="bq info">
- <p><b>Temporal <i class="amp">&</i> Geographical Ambiguity</b></p>
- <p>
- Despite seeming triviality of these ambiguities for the human comprehension, these represent a significant problem
- for the most deep learning neural networks unless specifically trained to resolve these particular cases. Procedural
- (deterministic) approach - as shown below - often yields a dramatically simpler and more robust solution, albeit the
- one that works only in a specific context.
- </p>
- </div>
- <p>
- Open <code>src/main/java/demo/<b>Weather.java</b></code> file and replace its content with the
- following code:
- </p>
- <pre class="brush: java, highlight: [30, 43, 130, 90, 132, 133, 134, 177]">
-package demo;
-
-import com.google.gson.Gson;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.nlpcraft.examples.weather.openweathermap.OpenWeatherMapException;
-import org.apache.nlpcraft.examples.weather.openweathermap.OpenWeatherMapService;
-import org.apache.nlpcraft.utils.keycdn.GeoManager;
-import org.apache.nlpcraft.utils.keycdn.beans.GeoDataBean;
-import org.apache.nlpcraft.model.NCIntent;
-import org.apache.nlpcraft.model.NCIntentMatch;
-import org.apache.nlpcraft.model.NCIntentSample;
-import org.apache.nlpcraft.model.NCIntentTerm;
-import org.apache.nlpcraft.model.NCModelFileAdapter;
-import org.apache.nlpcraft.model.NCRejection;
-import org.apache.nlpcraft.model.NCResult;
-import org.apache.nlpcraft.model.NCToken;
-
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static java.time.temporal.ChronoUnit.DAYS;
-
-public class WeatherModel extends NCModelFileAdapter {
- // System property for OpenWeatherMap API key.
- public final String OWM_API_KEY = "OWM_API_KEY";
-
- // Please register your own account at https://openweathermap.org/api and
- // replace this demo token with your own.
- // We are using the One Call API (https://openweathermap.org/api/one-call-api) in this example
- private final OpenWeatherMapService openWeather;
-
- private final GeoManager geoMrg = new GeoManager();
- private static final int DAYS_SHIFT_BACK = 5;
- private static final int DAYS_SHIFT_FORWARD = 7;
- private static final Gson GSON = new Gson();
- private static final Set<String> LOCAL_WORDS = new HashSet<>(Arrays.asList("my", "local", "hometown"));
-
- private Pair<Double, Double> prepGeo(NCIntentMatch ctx, Optional<NCToken> geoTokOpt) throws NCRejection {
- if (geoTokOpt.isPresent()) {
- NCToken geoTok = geoTokOpt.get();
-
- Map<String, Object> cityMeta = geoTok.meta("nlpcraft:city:citymeta");
-
- Double lat = (Double)cityMeta.get("latitude");
- Double lon = (Double)cityMeta.get("longitude");
-
- if (lat == null || lon == null) {
- String city = geoTok.meta("nlpcraft:city:city");
-
- throw new NCRejection(String.format("Latitude and longitude not found for: %s", city));
- }
-
- return Pair.of(lat, lon);
- }
-
- Optional<GeoDataBean> geoOpt = geoMrg.get(ctx.getContext().getRequest());
-
- if (geoOpt.isEmpty())
- throw new NCRejection("City cannot be determined.");
-
- // Manually process request for local weather. We need to separate between 'local Moscow weather'
- // and 'local weather' which are different. Basically, if there is word 'local/my/hometown' in the user
- // input and there is no city in the current sentence - this is a request for the weather at user's
- // current location, i.e. we should implicitly assume user's location and clear conversion context.
- // In all other cases - we take location from either current sentence or conversation STM.
-
- // NOTE: we don't do this separation on intent level as it is easier to do it here instead of
- // creating more intents with almost identical callbacks.
-
- @SuppressWarnings("SuspiciousMethodCalls")
- boolean hasLocalWord =
- ctx.getVariant().stream().anyMatch(t -> LOCAL_WORDS.contains(t.meta("nlpcraft:nlp:origtext")));
-
- if (hasLocalWord)
- // Because we implicitly assume user's current city at this point we need to clear
- // 'nlpcraft:city' tokens from conversation since they would no longer be valid.
- ctx.getContext().getConversation().clearStm(t -> t.getId().equals("nlpcraft:city"));
-
- // Try current user location.
- GeoDataBean geo = geoOpt.get();
-
- return Pair.of(geo.getLatitude(), geo.getLongitude());
- }
-
- @NCIntent(
- "intent=req " +
- "term~{tok_id() == 'wt:phen'}* " + // Zero or more weather phenomenon.
- "term(ind)~{" +
- "@isIndicator = has(tok_groups(), 'indicator') " + // Just to demo term variable usage.
- "@isIndicator" +
- "}* " + // Optional indicator words (zero or more).
- "term(city)~{tok_id() == 'nlpcraft:city'}? " + // Optional city.
- "term(date)~{tok_id() == 'nlpcraft:date'}?" // Optional date (overrides indicator words).
- )
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "Current forecast?",
- "Chance of rain in Berlin now?"
- })
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "Moscow forecast?",
- "Chicago history"
- })
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "What's the local weather forecast?",
- "What's the weather in Moscow?",
- "What's the current forecast for LA?",
- "What is the weather like outside?",
- "How's the weather?",
- "What's the weather forecast for the rest of the week?",
- "What's the weather forecast this week?",
- "What's the weather out there?",
- "Is it cold outside?",
- "Is it hot outside?",
- "Will it rain today?",
- "When it will rain in Delhi?",
- "Is there any possibility of rain in Delhi?",
- "Is it raining now?",
- "Is there any chance of rain today?",
- "Was it raining in Beirut three days ago?",
- "How about yesterday?"
- })
- public NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("ind") List<NCToken> indToksOpt,
- @NCIntentTerm("city") Optional<NCToken> cityTokOpt,
- @NCIntentTerm("date") Optional<NCToken> dateTokOpt
- ) {
- try {
- Instant now = Instant.now();
-
- Instant from = now;
- Instant to = now;
-
- if (indToksOpt.stream().anyMatch(tok -> tok.getId().equals("wt:hist")))
- from = from.minus(DAYS_SHIFT_BACK, DAYS);
- else if (indToksOpt.stream().anyMatch(tok -> tok.getId().equals("wt:fcast")))
- to = from.plus(DAYS_SHIFT_FORWARD, DAYS);
-
- if (dateTokOpt.isPresent()) { // Date token overrides any indicators.
- NCToken dateTok = dateTokOpt.get();
-
- from = Instant.ofEpochMilli(dateTok.meta("nlpcraft:date:from"));
- to = Instant.ofEpochMilli(dateTok.meta("nlpcraft:date:to"));
- }
-
- Pair<Double, Double> latLon = prepGeo(ctx, cityTokOpt); // Handles optional city too.
-
- double lat = latLon.getLeft();
- double lon = latLon.getRight();
-
- return NCResult.json(GSON.toJson(from == to ? openWeather.getCurrent(lat, lon) : openWeather.getTimeMachine(lat, lon, from, to)));
- }
- catch (OpenWeatherMapException e) {
- throw new NCRejection(e.getLocalizedMessage());
- }
- catch (NCRejection e) {
- throw e;
- }
- catch (Exception e) {
- throw new NCRejection("Weather provider error.", e);
- }
- }
-
- /**
- * Loads the model.
- */
- public WeatherModel() {
- // Load model from external JSON file on classpath.
- super("weather_model.json");
-
- // Try system variable first.
- String apiKey = System.getProperty(OWM_API_KEY);
-
- if (apiKey == null)
- // Try environment variable next.
- apiKey = System.getenv(OWM_API_KEY);
-
- if (apiKey == null)
- throw new OpenWeatherMapException(String.format("Provide OpenWeatherMap API key using '-D%s=<your-key-here>' system property.", OWM_API_KEY));
-
- openWeather = new OpenWeatherMapService(apiKey, 5, 7);
- }
-
- @Override
- public void onDiscard() {
- openWeather.stop();
- }
-}
- </pre>
- <ul>
- <li>
- <code>Line 177</code> loads the model configuration from the external <code>weather_model.json</code>
- file.
- </li>
- <li>
- Method <code>preGeo(...)</code> on the <code>line 43</code> handles the geolocation processing including
- IP-based geo-location and resolution of the geographical ambiguity.
- </li>
- <li>
- <code>Line 130</code> defines a callback for the intent defined on the <code>line 90</code>. Note that
- callback implementation also deals with the temporal ambiguity.
- </li>
- <li>
- <code>Lines 132-134</code> define formal callback method parameters that correspond
- to the intent's terms (see <code>line 90</code>).
- </li>
- <li>
- <code>Lines 101-129</code> define input sentence samples that are used for both documentation and
- auto-validation purposed (see <a href="#testing">testing</a> second for details).
- </li>
- <li>
- Make sure to provide you <a target="new" href="https://openweathermap.org/api/one-call-api">OpenWeather API</a>
- key on the <code>line 30</code>.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Once we have our model ready let's go to <code>~/Weather</code> directory and run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/Weather
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component that you generally need to start it only once.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Remember the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our model code next to intent definition?
- </p>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/Weather/target/classes --mdls=demo.Weather
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- <p>
- Look at the output of this command and you will see the test results for all our sample utterances:
- </p>
- <p>
- <img style="max-width: 871px !important;" class="img-fluid" alt="" src="/images/weather-bot-test.png">
- </p>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created weather bot data model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#new_project">New Project</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/getting-started.html b/getting-started.html
index c4c6d8c..e88aa3b 100644
--- a/getting-started.html
+++ b/getting-started.html
@@ -48,12 +48,6 @@
</ul>
</section>
<section>
- <h2 class="section-title">Weather Forecast <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We'll be testing <a href="/examples/weather_bot.html">Weather Example</a>
- to ask questions about weather forecast using REST APIs. This example returns a JSON weather
- data for variety of different inquiries about the past, present or future weather conditions.
- </p>
<h2 id="probe-server" class="section-sub-title">Data Probe <span class="amp">&</span> REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
<a href="/server-and-probe.html">Data probes</a> are used to deploy and host data model, while <a href="/server-and-probe.html">REST server</a> (or a
diff --git a/metrics-and-tracing.html b/metrics-and-tracing.html
deleted file mode 100644
index d8cc82a..0000000
--- a/metrics-and-tracing.html
+++ /dev/null
@@ -1,307 +0,0 @@
----
-active_crumb: Metrics <span class="amp">&</span> Tracing
-layout: documentation
-id: metrics
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div id="metrics-and-tracing" class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both REST server and data probe come with <a target=_ href="https://opencensus.io">OpenCensus</a>
- instrumentation. NLPCraft bundles several built-in exporters for various OpenCensus trace and stats
- backends:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Backend</th>
- <th>Built-In Exporters</th>
- <th>Stats or Trace?</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td>
- <a target=_ href="https://www.jaegertracing.io/">
- <img alt="" style="width: 48px" src="/images/jaeger-logo-x.png">
- <div class="exporter-name">Jaeger</div>
- </a>
- </td>
- <td>
- Server: <code>NCJaegerExporter</code><br>
- Probe:
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCJaegerExporter.html">NCJaegerExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://zipkin.io/">
- <img alt="" style="width: 48px" src="/images/zipkin-logo.jpg">
- <div class="exporter-name">Zipkin</div>
- </a>
- </td>
- <td>
- Server: <code>NCZipkinExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCZipkinExporter.html">NCZipkinExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://prometheus.io/">
- <img alt="" style="width: 48px" src="/images/prometheus-logo-x.png">
- <div class="exporter-name">Prometheus</div>
- </a>
- </td>
- <td>
- Server: <code>NCPrometheusExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://grafana.com/">
- <img alt="" style="width: 48px" src="/images/grafana-logo-x.png">
- <div class="exporter-name">Grafana<sup>*</sup></div>
- </a>
- </td>
- <td>
- Server: <code>NCPrometheusExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://cloud.google.com/stackdriver/">
- <img alt="" style="width: 48px" src="/images/stackdriver-stats-x.png">
- <div class="exporter-name">Stackdriver</div>
- </a>
- </td>
- <td>
- Server: <code>NCStackdriverStatsExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverStatsExporter.html">NCStackdriverStatsExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://cloud.google.com/stackdriver/">
- <img alt="" style="width: 48px" src="/images/stackdriver-trace-x.png">
- <div class="exporter-name">Stackdriver</div>
- </a>
- </td>
- <td>
- Server: <code>NCStackdriverTraceExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverTraceExporter.html">NCStackdriverTraceExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- </tbody>
- </table>
- <p>
- <sup><b>NOTE:</b> (*) Grafana works through Prometheus integration.</sup>
- </p>
- <p>
- All built-in exporters are a simple adaptation of the standard OpenCensus <a target=_ href="https://opencensus.io/exporters/supported-exporters/java/">exporters</a>
- to life cycle components for the
- REST server or the data probe. Each exporter has a few lines of code and you can create your own
- exporters for other backends.
- </p>
- </section>
- <section id="server">
- <h2 class="section-title">Configuring Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- OpenCensus exporters for the REST server are lifecycle components. Server lifecycle components are the implementations
- of <code>NCServerLifecycle</code> trait. Lifecycle components get callbacks for various stages of the server
- lifecycle and thus can register and unregister "primordial" components such as OpenCensus exporters.
- </p>
- <p>
- REST server lifecycle components are specified in <code>nlpcraft.server.lifecycle</code>
- <a href="server-and-probe.html">configuration</a> property. For example, for file based configuration:
- </p>
- <pre class="brush: js, highlight: [17, 18]">
-nlpcraft {
- server {
- ...
- lifecycle = [
- "org.apache.nlpcraft.server.lifecycle.opencensus.NCPrometheusExporter",
- "org.apache.nlpcraft.server.lifecycle.opencensus.NCJaegerExporter"
- ]
- ...
- }
-}
- </pre>
- <p>
- Note that each server exporter has default configuration that can be overridden via
- <a href="server-and-probe.html">configuration</a> properties or environment variables. Note also that all
- built-in <b>trace exporters</b> use "always" sampling by default that is suitable
- only for demo or development purposes. For production usage you will need to modify the exporter
- to provide more effective <a target=_ href="https://opencensus.io/tracing/sampling/">sampling strategy</a>.
- </p>
- </section>
- <section id="probe">
- <h2 class="section-title">Configuring Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Just like on the REST server OpenCensus exporters for the data probe are lifecycle components. Probe lifecycle
- components are the implementations
- of <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCLifecycle.html">org.apache.nlpcraft.model.NCLifecycle</a>
- interface. Lifecycle components get callbacks for various stages of the data probe
- lifecycle and thus can register and unregister "primordial" components such as OpenCensus exporters. For example, using
- file based configuration:
- </p>
- <pre class="brush: js, highlight: [14, 15]">
-nlpcraft {
- probe {
- ...
- lifecycle = [
- "org.apache.nlpcraft.model.opencensus.NCPrometheusExporter",
- "org.apache.nlpcraft.model.opencensus.NCJaegerExporter"
- ]
- ...
- }
-}
- </pre>
- <p>
- Just like with server exporters, each probe exporter has default configuration that can be overridden via
- <a href="server-and-probe.html">configuration</a> properties or environment variables. Note also that all
- built-in <b>trace exporters</b> use "always" sampling by default that is suitable
- only for demo or development purposes. For production usage you will need to modify the exporter
- to provide more effective <a target=_ href="https://opencensus.io/tracing/sampling/">sampling strategy</a>.
- Please consult individual exporter classes:
- </p>
- <ul>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCJaegerExporter.html">NCJaegerExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCZipkinExporter.html">NCZipkinExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverStatsExporter.html">NCStackdriverStatsExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverTraceExporter.html">NCStackdriverTraceExporter</a></li>
- </ul>
- </section>
- <section id="traces">
- <h2 class="section-title">Traces <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When traces are exporter, there are two service names used for NLPCraft traces that will appear on UI backends:
- </p>
- <ul>
- <li><code>nlpcraft-server</code> for traces from REST server.</li>
- <li><code>nlpcraft-probe</code> for traces from data probes.</li>
- </ul>
- <p>
- All traces are extensively tagged with various information. Specifically, watch out for <code>srvReqId</code>,
- <code>modelId</code> and <code>userId</code> tags that will help you navigate across traces pertaining to specific
- request, models or users.
- </p>
- </section>
- <section id="metrics">
- <h2 class="section-title">Metrics <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When it comes to metrics each exporter has different set of configuration for either a namespace or
- name prefix on how these metrics will be available on the UI backends:
- </p>
- <ul>
- <li>
- <a target=_ href="https://prometheus.io/">Prometheus</a> and <a target=_ href="https://grafana.com">Grafana</a>
- stats exporters by default use <code>nlpcraft-server</code> and <code>nlpcraft-probe</code> namespaces.
- </li>
- <li>
- <a target=_ href="https://cloud.google.com/stackdriver/">Stackdriver</a> stats exporters use default
- <code>custom.googleapis.com/nlpcraft/server</code> and <code>custom.googleapis.com/nlpcraft/probe</code> metric
- name prefixes.
- </li>
- </ul>
- <p>
- <b>NOTE:</b> default namespaces and service prefixes can be changed via <a href="server-and-probe.html">configuration</a>
- properties or environment variables
- </p>
- <p>
- The following metrics are generated by the REST server:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Name <sub>without prefix or namespace</sub></th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- {% for api in site.data.metrics.server %}
- <tr>
- <td>
- <code>{{api.name}}_latdist_bucket</code><br>
- <code>{{api.name}}_latdist_count</code><br>
- <code>{{api.name}}_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the <b>{{api.description}}</b> latencies.</td>
- </tr>
- {% endfor %}
- <tr>
- <td>
- <code>roundtrip_latdist_bucket</code><br>
- <code>roundtrip_latdist_count</code><br>
- <code>roundtrip_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the full server-probe-server round trip latencies.</td>
- </tr>
- </tbody>
- </table>
- <p>
- The following metrics are generated by the data probe:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Name <sub>without prefix or namespace</sub></th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- {% for api in site.data.metrics.probe %}
- <tr>
- <td>
- <code>{{api.name}}_latdist_bucket</code><br>
- <code>{{api.name}}_latdist_count</code><br>
- <code>{{api.name}}_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the <b>{{api.description}}</b> latencies.</td>
- </tr>
- {% endfor %}
- </tbody>
- </table>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#server">Configuring Server</a></li>
- <li><a href="#probe">Configuring Probe</a></li>
- <li><a href="#traces">Traces</a></li>
- <li><a href="#metrics">Metrics</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
diff --git a/server-and-probe.html b/server-and-probe.html
deleted file mode 100644
index d2e611c..0000000
--- a/server-and-probe.html
+++ /dev/null
@@ -1,618 +0,0 @@
----
-active_crumb: Server <span class="amp">&</span> Probe
-layout: documentation
-id: server_and_probe
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div id="server-and-probes" class="col-md-8 second-column">
- <section>
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server and data probe are the two runtime components that you need to run when using NLPCraft.
- Data probes are used to deploy and host data model, while REST server (or a cluster of servers) is used
- to accept client REST call and route them to the data models via data probes.
- </p>
- <h2 class="section-sub-title">REST Server vs. Data Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- It's important to remember why REST server is a separate component from a data probe. While a
- typical deployment would have only one REST server (or a cluster of REST servers behind a single
- load balancer), there are maybe multiple data probes hosting different data models deployed in
- different physical locations, managed through different life cycles and requiring different
- security and network configurations.
- </p>
- <p>
- Moreover, REST server is a heavy and resource consuming component that is built around Apache
- Ignite distributing in-memory computing capabilities - while the data probe is a lightweight
- data model container. During the development and testing of data models, the developers need to
- frequently redeploy data models by restarting the data probe. If the REST server and the data probe
- would be one component - this process would be very inefficient.
- </p>
- <div class="bq success">
- <div class="bq-idea-container">
- <div><div>💡</div></div>
- <div>
- <p><b>Configuration</b></p>
- <p>
- Both REST server and the data probe can share their configuration file or be configured
- individually. Read more about this in <a href="#config">configuration</a>
- section.
- </p>
- </div>
- </div>
- </div>
- <h2 class="section-sub-title">All-Inclusive JAR <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- <a href="/download.html#zip">Binary</a> NLPCraft ZIP download comes with a single executable JAR file that includes all
- necessary dependencies (except for examples): <code>build/<b>apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar</b></code>.
- This single all-inclusive JAR file can be used to start any NLPCraft runtime components as standard
- Java applications:
- </p>
- <ul>
- <li><a href="#server">REST Server</a></li>
- <li><a href="#probe">Data Probe</a></li>
- <li><a href="/tools/script.html">NLPCraft CLI</a></li>
- <li><a href="/data-model.html">Model API</a></li>
- </ul>
- <p>
- Note that if you downloaded the <a href="/download.html#src">source</a> ZIP you need to run <code class="script">mvn clean package -P examples</code> to
- get the <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code>
- file. It will be located in <code>nlpcraft/target</code> sub-folder.
- </p>
- <h2 class="section-sub-title">Examples JARs <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- If you downloaded binary release the examples JARs are pre-built and shipped within it. They are located in <code>build/nlpcraft-examples/xxx</code> folder
- for each <code>xxx</code> example. If you downloaded a source release you will need to run <code class="script">mvn clean package -P examples</code>
- and examples JARs will be located in each individual module under its <code>target</code> sub-folder.
- </p>
- </section>
- <section id="server">
- <h2 class="section-title">REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server accepts client REST calls and routes them to the data model hosted by data probes.
- REST server can be started in different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-docker" role="tab"><i class="fab fa-docker"></i> Docker</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-srv-class" role="tabpanel">
- <p></p>
- <p>
- If using executable JAR:
- </p>
- <pre class="brush: bash">
- $ java -Xms1024m -jar apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar -server
- </pre>
- <p>
- If specifying additional classpath components and need <code>-cp</code> parameter:
- </p>
- <pre class="brush: bash">
- $ java -Xms1024m -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.NCStart -server
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Make sure to provide correct path to <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> is a common entry point for all NLPCraft runtime components and
- can be used to start REST server from IDE.
- </li>
- </ul>
- <p>
- <b>Parameters</b>:
- </p>
- <dl>
- <dt>
- <code>-server</code>
- </dt>
- <dd>
- <em>Mandatory</em> parameter to indicate that you are starting the REST server.
- </dd>
- <dt><code>-config=path</code></dt>
- <dd>
- <em>Optional</em> parameter to provide configuration file path.
- Server will automatically look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file. If the configuration
- file has different name or in different location use <code>-config=path</code> parameter
- where <code>path</code> is an absolute path to the configuration file. Note that the server and the data
- probe can use the same file for their configuration (like the
- default <code>nlpcraft.conf</code> contains configuration for both the server and the data probe).
- </dd>
- <dt><code>-igniteConfig=path</code></dt>
- <dd>
- <em>Optional</em> parameter to provide <a target=_ href="https://ignite.apache.org/">Apache Ignite</a> configuration file path.
- Note that Apache Ignite is used as a cluster computing plane and a default distributed storage.
- Server will automatically look for <code>ignite.xml</code>
- configuration file in the same directory as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- If the configuration file has different name or in different location use <code>-igniteConfig=path</code> parameter
- where <code>path</code> is an absolute path to the Ignite configuration file.
- </dd>
- </dl>
- <p>
- <b>VM Options</b>:
- </p>
- <p>
- NLPCraft REST server uses Apache Ignite 2.x as its distributed in-memory computing plane. Apache Ignite
- <a target=_new href="https://apacheignite.readme.io/docs/getting-started#running-ignite-with-java-11-and-later-versions">requires</a>
- the following additional JVM options to be used when running Apache Ignite 2.x on JDK 11:
- </p>
- <pre class="brush: text">
- --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
- --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
- --add-opens=java.base/java.nio=ALL-UNNAMED
- --add-opens=java.base/java.io=ALL-UNNAMED
- --add-opens=java.base/java.util=ALL-UNNAMED
- --add-opens=java.base/java.lang=ALL-UNNAMED
- --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED
- --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED
- --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED
- --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
- --illegal-access=permit
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script automatically uses
- these options for <code>start-server</code> command.
- </li>
- </ul>
- <div class="bq success">
- Since this list of required VM options is long it is rather impractical and inconvenient to use it from the
- command line. It is <b>highly recommended</b> to either use IDE or <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script
- to manage REST server lifecycle.
- </div>
- </div>
- <div class="tab-pane fade show" id="nav-srv-docker" role="tabpanel">
- <p></p>
- <p>
- If Docker image is available for given version you can start REST server as follows:
- </p>
- <pre class="brush: bash">
- $ docker run -m 8G -p 8081:8081 -p 8201:8201 -p 8202:8202 nlpcraftserver/server:{{site.latest_version}}
- </pre>
- </div>
- </div>
- <h2 class="section-sub-title">JVM Memory <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Make sure to allocate enough memory for server JVM using <code>-Xms</code> JVM option, i.e. <code>-Xms1024m</code>.
- Many 3rd party NLP engines like Stanford CoreNLP are very memory intensive and may require several GBs
- of JVM heap allocated depending on the models used. Note that when server JVM has insufficient heap
- memory the Apache Ignite may throw the following warning logs:
- </p>
- <pre class="brush: text">
- Jul-22 13:27:56 [INFO ] ...
- Jul-22 13:28:08 [WARN ] Possible too long JVM pause: 11364 milliseconds.
- Jul-22 13:28:11 [INFO ] ...
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script automatically uses
- <code>-Xms1024m</code> for <code>start-server</code> command.
- </li>
- </ul>
- <p>
- The abnormally long GC pauses (over 5s) can be caused by the excessive memory swapping performed by OS due to
- insufficient JVM heap memory.
- </p>
- </section>
- <section id="probe">
- <h2 class="section-title">Data Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Data probes are used to deploy and host data mode, and can also be started in several ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-probe-script" role="tab" >NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-probe-class" role="tab" >Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-probe-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-probe --cp=/path/to/model/classes # Use default configuration.
- $ bin/nlpcraft.sh start-probe --cp=/path/to/model/classes --mdls=com.package.MyModel
- > restart-probe # Restart the probe with the last set of parameters in REPL mode.
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- <code>--cp</code> parameter must provide additional JVM classpath for models to deploy in this probe.
- </li>
- <li>
- Optional <code>--mdls</code> parameter can be used to specify a one or more specific models to deploy if more than
- one model is available.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-probe</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-probe-class" role="tabpanel">
- <p></p>
- <p>
- If using executable JAR:
- </p>
- <pre class="brush: bash">
- $ java -jar apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar -probe
- </pre>
- <p>
- If specifying additional classpath components and need <code>-cp</code> parameter:
- </p>
- <pre class="brush: bash">
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar:/my/project/classes org.apache.nlpcraft.NCStart -probe -config=/my/project/probe.conf
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <code>/my/project</code> directory contains user-defined model implementation
- </li>
- <li>
- Make sure to provide correct path to <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> is a common entry point for all NLPCraft runtime components.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> should be used to star data probe from IDE.
- </li>
- </ul>
- <p>
- <b>Parameters:</b>
- </p>
- <dl>
- <dt>
- <code>-probe</code>
- </dt>
- <dd>
- <em>Mandatory</em> parameter to indicate that you are starting a data probe.
- </dd>
- <dt><code>-config=path</code></dt>
- <dd>
- <p>
- <em>Optional</em> parameter to provide probe configuration file path.
- Data probe will automatically look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file. If the configuration
- file has different name or in different location use <code>-config=path</code> parameter
- where <code>path</code> is an absolute path to the data probe configuration file. Note that the server and the data
- probe can use the same file for their configuration (like the
- default <code>nlpcraft.conf</code> contains configuration for both the server and the data probe).
- </p>
- </dd>
- </dl>
- </div>
- </div>
- </section>
- <section id="config">
- <h2 class="section-title">Configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both REST server and the data probe use <a target=_ href="https://github.com/lightbend/config/">Typesafe Config</a> for their configuration:
- </p>
- <ul>
- <li>Both the server and the data probe come with default configuration available in <code>build/<b>nlpcraft.conf</b></code> file.</li>
- <li>
- By default, REST server is looking for <code>server.conf</code> and then <code>nlpcraft.conf</code> configuration file.
- </li>
- <li>
- By default, data probe is looking for <code>probe.conf</code> and then <code>nlpcraft.conf</code> configuration file.
- </li>
- <li>Custom configuration or default overrides can be placed into a file or provided via environment variables.</li>
- <li>Configuration files use <a target=_ href="https://github.com/lightbend/config/blob/master/HOCON.md">HOCON</a> file format.</li>
- <li>
- When server and probe use different configuration files - each file would have either <code>nlpcraft.server</code>
- or <code>nlpcraft.probe</code> sub-section.
- </li>
- </ul>
- <p>
- By default, when REST server or data probe start they look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file and the on their classpath. You can change this behavior with
- <code>-config=path</code> parameter.
- </p>
- <div class="bq success">
- <div class="bq-idea-container">
- <div><div>💡</div></div>
- <div>
- <p><b>Configuration Example</b></p>
- <p>
- Default configuration is available in <code>build/<b>nlpcraft.conf</b></code> file and it is extensively
- documented including all optional parameters and default values.
- </p>
- </div>
- </div>
- </div>
- <h2 class="section-sub-title">Server <span class="amp">&</span> Probe Configurations <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Default configuration is available in <code>build/<b>nlpcraft.conf</b></code> file and it is extensively documented. It has subsections
- for the server and probe configuration. You can also separate server and probe configurations into their own
- separate files. While server and probe can use any file name for their configuration files, the server
- looks for <code>server.conf</code> file by default, and the probe looks for <code>probe.conf</code>
- file before searching for <code>nlpcraft.conf</code> file.
- Each such file would have a subsection of configuration for either server or data probe.
- </p>
- <p>
- Server configuration file (e.g. <code>server.conf</code>):
- </p>
- <pre class="brush: js">
-nlpcraft {
- server {
- ...
- }
-}
- </pre>
- <p>
- Probe configuration file (e.g. <code>probe.conf</code>):
- </p>
- <pre class="brush: js">
-nlpcraft {
- probe {
- ...
- }
-}
- </pre>
- </section>
- <section id="override">
- <h2 class="section-sub-title">Custom Configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- While you can change configuration file or files for your own needs (and use <code>-config=...</code>
- parameter described above to provide path to that file) it is often more convenient to use the default configuration file and
- change one or two properties in it. You can accomplish this by using standard
- <a target=_ href="https://github.com/lightbend/config#optional-system-or-env-variable-overrides">HOCON overriding</a>
- via environment variables:
- </p>
- <ol>
- <li>
- Set probe or server JVM system property <code>-Dconfig.override_with_env_vars=true</code> which will instruct
- configuration framework to look for external overrides.
- </li>
- <li>For each configuration property <code>x.y.z</code> set the overriding environment variable <code>CONFIG_FORCE_x_y_z=some_value</code></li>
- <li>See more details on <a target=_ href="https://github.com/lightbend/config#optional-system-or-env-variable-overrides">HOCON documentation</a>.</li>
- </ol>
- <p>
- Consider the following snippet of NLPCraft configuration:
- </p>
- <pre class="brush: js">
-nlpcraft {
- probe {
- models = "com.nlp.MyModel"
- }
-
- server {
- lifecycle = "org.apache.nlpcraft.server.lifecycle.opencensus.NCJaegerExporter"
-
- rest {
- host = "0.0.0.0"
- port = 8081
- apiImpl = "org.apache.nlpcraft.server.rest.NCBasicRestApi"
- }
- }
-}
- </pre>
- <p>
- You can override these properties with the following environment variables:
- </p>
- <p>
- <code>CONFIG_FORCE_<b>nlpcraft_server_rest_host</b>=1.2.3.4</code><br>
- <code>CONFIG_FORCE_<b>nlpcraft_server_lifecycle</b>="org.nlp.Lifecycle1, org.nlp.Lifecycle1"</code><br>
- <code>CONFIG_FORCE_<b>nlpcraft_probe_models</b>="com.nlp.MyModel, com.nlp.AnotherModel"</code>
- </p>
- <div class="bq info">
- <b>Examples</b>
- <p>
- Note that all examples that come with NLPCraft have instructions that use environment variable overriding
- for running their data probes. They use default <code>nlpcraft.conf</code> file and override
- one <code>nlpcraft.probe.models</code> property (see above) to specify what model the data probe
- needs to deploy.
- </p>
- </div>
- </section>
- <section id="ansi">
- <h2 class="section-sub-title">ANSI Colors <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both NLPCraft server and probe use ANSI coloring via
- <a target=_ href="https://en.wikipedia.org/wiki/ANSI_escape_code">ANSI escape sequences</a> for their log
- output by default. ANSI coloring provides easer console log comprehension and modern esthetics:
- </p>
- <p>
- <img class="non-fluid-img" src="/images/ansi_colors.png" alt="">
- </p>
- <p>
- However, there are
- cases when either specific console does not support ANSI escape sequences, or specific color schema
- isn't suitable or log being redirected to a file or piped to downstream system. In these cases you need to
- disable ANSI coloring to avoid polluting log with unprocessed ANSI escape codes.
- </p>
- <p>
- You can disable ANSI coloring in either server, probe or both by supplying the following system
- property to JVM process: <code>-D<b>NLPCRAFT_ANSI_COLOR_DISABLED</b>=true</code>
- </p>
- </section>
- <section id="testing">
- <h2 class="section-title">CI Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- It is a good practice to run units tests during routine builds using Maven (or others CI toolchains). To
- test data models you need to have a running server and then start one or more data probes with
- models you want to test. While doing this from IDE can be trivial enough, doing this from Maven
- can be tricky.
- </p>
- <p>
- The challenge is that from the Maven build you need to start the server, wait til its fully started and
- initialized, and only then start issuing REST calls, start data probes or run tests that use <a href="/tools/embedded_probe.html">embedded probes</a>.
- When done manually (e.g. from IDE) you can visually observe when the server finished its startup and then manually
- launch the tests. In Maven, however, you need to use a special plugin to accomplish the same in
- automated fashion.
- </p>
- <div class="bq info">
- <b>Probe Waits For Server</b>
- <p>
- Technically, when a data probe starts up it will initialize, load the models, and will automatically wait for the server to get online
- if it isn't yet (as well as periodically check for it). Once server is online the data probe will automatically connect to it. However,
- if the unit tests don't use data probe and only issue REST calls then these tests have to somehow wait for the
- server to get online.
- </p>
- <p>
- To overcome this challenge you can use
- <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>
- Maven plugin.
- </p>
- </div>
- <p>
- To get around this problem NLPCraft uses <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>
- Maven plugin in its own build. This plugin allows to start the external process and use configured URL
- endpoint to check whether or not the external process has fully started. This works perfect with NLPCraft server
- <a href="/using-rest.html">health check REST call</a>. The plugin can be configured in the following way for your own project
- (taken directly from NLPCraft <a target="github" href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft/pom.xml"><code>pom.xml</code></a>):
- </p>
- <pre class="brush: xml, highlight: [14, 16, 32]">
-<plugin>
- <groupId>com.bazaarvoice.maven.plugins</groupId>
- <artifactId>process-exec-maven-plugin</artifactId>
- <version>0.9</version>
- <executions>
- <execution>
- <id>pre-integration-test</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>start</goal>
- </goals>
- <configuration>
- <name>server</name>
- <healthcheckUrl>http://localhost:8081/api/v1/health</healthcheckUrl>
- <waitAfterLaunch>180</waitAfterLaunch>
- <processLogFile>${project.build.directory}/server.log</processLogFile>
- <arguments>
- <argument>java</argument>
- <argument>-Xmx4G</argument>
- <argument>-Xms4G</argument>
- <argument>--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.nio=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.io=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.util=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.lang=ALL-UNNAMED</argument>
- <argument>--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED</argument>
- <argument>--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED</argument>
- <argument>--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</argument>
- <argument>--illegal-access=permit</argument>
- <argument>-DNLPCRAFT_ANSI_COLOR_DISABLED=true</argument>
- <argument>-Djdk.tls.client.protocols=TLSv1.2</argument>
- <argument>-jar</argument>
- <argument>${project.build.directory}/${nlpcraft.all.deps.jar}</argument>
- <argument>-server</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>stop-all</id>
- <phase>post-integration-test</phase>
- <goals>
- <goal>stop-all</goal>
- </goals>
- </execution>
- </executions>
-</plugin>
- </pre>
- <p>
- <b>NOTES</b>:
- </p>
- <ul>
- <li>
- On line 14 we specify the URL endpoint to check whether or not our server is online. We use
- <code>/health</code> localhost REST call for that.
- </li>
- <li>
- On line 16 we redirect the output from server to a dedicated file to <b>avoid interleaving</b> log
- from server and log from data probe in the same console (where we are running the Maven build from).
- Such interleaving will make the combined log unreadable and can cause output problem for the console
- due to mixed up ANSI escape sequences from server and data probe.
- </li>
- <li>
- Since the server log is piped out to a separate file we disable ANSI coloring for the server
- on the line 32.
- </li>
- </ul>
- <div class="bq info">
- <b>Avoid Interleaving Logs</b>
- <p>
- When running both server and the data probe(s) from the Maven build it is important to avoid interleaving
- logs from the server and the probe. Such interleaving will make the combined log in Maven unreadable and
- can cause console malfunction due to mixed up ANSI escape codes. It is idiomatic in such cases to:
- </p>
- <ul>
- <li>
- Disable ANSI coloring for the server via <code>-D<b>NLPCRAFT_ANSI_COLOR_DISABLED</b>=true</code>
- </li>
- <li>
- Pipe out the server log into a dedicated file using Maven plugins like
- <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>.
- </li>
- </ul>
- </div>
- <br/>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#server">REST Server</a></li>
- <li><a href="#probe">Data Probe</a></li>
- <li><a href="#config">Configuration</a></li>
- <li><a href="#testing">CI Testing</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
diff --git a/tools/embedded_probe.html b/tools/embedded_probe.html
deleted file mode 100644
index b1a30a9..0000000
--- a/tools/embedded_probe.html
+++ /dev/null
@@ -1,159 +0,0 @@
----
-active_crumb: Embedded Probe
-layout: documentation
-id: embedded_probe
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Embedded probe allows to run data probe in the same JVM process as JVM-based client application.
- </p>
- <p>
- Typically, data probes are launched in their own independent JVMs. However, when a client application is
- itself a JVM-based process, it is sometimes preferable for performance reasons to host
- a data model - and hence the data probe - in the same "client" JVM.
- </p>
- <p>
- In a standard processing flow, when data probe (with its deployed data models) is hosted in its own independent JVM,
- the request from
- the client application takes 4 network hops to go server, then to the data probe and back to the client application (see fig 1. below):
- </p>
- <figure>
- <img class="img-fluid" style="max-width: 500px !important;" src="/images/emb_probe1.png" alt="">
- <figcaption><b>Fig 1.</b> Standard Processing Flow</figcaption>
- </figure>
- <p>
- When performing model unit testing or when client "application" can be combined with the data probe in a single JVM, the
- embedded probe mode provides significant performance advantage. Using embedded probe you can reduce number
- of network hops to 2 in a similar scenario - in which case the <b>hop<sub>3</sub></b> and
- <b>hop<sub>4</sub></b> (see fig. 1) are performed inside of JVM between client and embedded probe:
- </p>
- <figure>
- <img class="img-fluid" style="max-width: 500px !important;" src="/images/emb_probe2.png" alt="">
- <figcaption><b>Fig 2.</b> Embedded Processing Flow</figcaption>
- </figure>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Embedded probe support consists of the following types in <code>org.apache.nlpcraft.probe.embedded</code> package:
- </p>
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/embedded/NCEmbeddedProbe.html">
- NCEmbeddedProbe
- </a> - Embedded probe main class.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/embedded/NCEmbeddedResult.html">
- NCEmbeddedResult
- </a> - Processing result container since result is delivered as a local JVM call.
- </li>
- </ul>
- <p>
- Here's Java code snippet from <a href="/examples/alarm_clock.html">Alarm Clock</a>
- example illustrating the usage of embedded probe together with NLPCraft test framework and JUnit 5:
- </p>
- <pre class="brush: java, highlight: [6, 18]">
-public class AlarmTest {
- private NCTestClient cli;
-
- @BeforeEach
- void setUp() throws NCException, IOException {
- if (NCEmbeddedProbe.start(AlarmModel.class)) {
- cli = new NCTestClientBuilder().newBuilder().build();
-
- cli.open("nlpcraft.alarm.ex");
- }
- }
-
- @AfterEach
- void tearDown() throws NCException, IOException {
- if (cli != null)
- cli.close();
-
- NCEmbeddedProbe.stop();
- }
-
- @Test
- public void test() throws NCException, IOException {
- // Should be passed.
- assertTrue(cli.ask("Ping me in 3 minutes").isOk());
- assertTrue(cli.ask("Buzz me in an hour and 15mins").isOk());
- assertTrue(cli.ask("Set my alarm for 30s").isOk());
- }
-}
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Lines 6 and 18 show the embedded data probe start and stop as it's used in the unit tests.
- </li>
- </ul>
- <p>
- It's important to note that there are number of inherent limitations associated with embedded probe:
- </p>
- <ul>
- <li>
- Embedded probe is only available for JVM-based applications (and can be used with any JVM languages).
- </li>
- <li>
- There can be only one embedded probe per JVM.
- </li>
- <li>
- Once data probe is stopped and cannot be re-started again in the same JVM.
- </li>
- <li>
- Even though the caller can register local-JVM listener for the query results, these results
- will still be asynchronously delivered to the REST server in the usual manner so that other clients
- could fetch these results to maintain internal logging, tracing and metrics. If the client
- application hosting data model and its probe <i>is the only client</i> for that model it needs to cancel the
- request on the REST server after receiving a local-JVM callback to release associated
- resources on the REST server.
- </li>
- </ul>
- <p>
- Note that embedded probe is automatically used by the <a href="/tools/test_framework.html#autotest">auto model validator</a>.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/script.html b/tools/script.html
deleted file mode 100644
index 30e85e3..0000000
--- a/tools/script.html
+++ /dev/null
@@ -1,280 +0,0 @@
----
-active_crumb: Command Line
-layout: documentation
-fa_icon: fa-tools
-id: script
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="tldr">
- <h2 class="section-title">TL;DR; <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Watch this quick video (12:05) for introduction to NLPCraft CLI:
- </p>
- <div>
- <iframe
- width="514"
- height="289"
- src="https://www.youtube.com/embed/LNfny60bmgg?modestbranding=1"
- title="NLPCraft - Command Line Utility"
- frameborder="0"
- allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
- allowfullscreen>
- </iframe>
- </div>
- </section>
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft CLI <code>bin/nlpcraft.{sh|cmd}</code> script is a <b>central management utility</b> for NLPCraft that supports
- interactive REPL and command line modes. Most of the functions this script does can be done through running
- Java applications or using REST API - yet using this script you can perform the same tasks much simpler and quicker.
- From starting and managing REST server and probes, running various built-in tools, to testing models via REST API -
- NLPCraft CLI provides a single centralized utility that makes a quick work out
- of all these routine tasks.
- </p>
- <p>
- The script is only available in either official Apache <a href="/download.html#src">source ZIP</a> or in
- <a href="/download.html#zip">binary ZIP</a> distributions. If you used Maven or similar build tools to
- add NLPCraft to your project as a dependency, this script will not be available out-of-the-box - you will
- need to download ZIP distribution separately.
- </p>
- <div class="bq warn">
- <b>ZIP Distribution</b>
- <p>
- The NLPCraft CLI is only available in <a href="/download.html">ZIP distributions</a>. For Maven/Gradle/SBT
- projects you will need to download <a href="/download.html">ZIP distribution</a> separately.
- </p>
- </div>
- <p>
- The script is located in <code>/bin</code> sub-folder of the installation root and is available as
- <code>nlpcraft.sh</code> for <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i> environments. Both versions provide identical functionality.
- </p>
- </section>
- <section id="quick_start">
- <h2 class="section-title">Quick Start <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- The <code>bin/nlpcraft.{sh|cmd}</code> works in two modes:
- </p>
- <ul>
- <li>
- Command line mode where it executes a single command at a time.
- </li>
- <li>
- Interactive REPL mode. To enter REPL mode run <code>bin/nlpcraft.{sh|cmd}</code> without arguments.
- </li>
- </ul>
- <p>
- All the functionality and command syntax remain the same regardless of whether a command is executed from a command line or
- in REPL mode.
- </p>
- <p>
- To get started, run <code>bin/nlpcraft.{sh|cmd}</code>:
- </p>
- <p>
- <img class="img-fluid" src="/images/cli1.png" alt="">
- </p>
- <p>
- Script starts in the REPL mode by default.
- </p>
- <div class="bq info">
- <b>REPL Prompt</b>
- <p>
- REPL prompt shows 4 pieces of information:
- </p>
- <ul>
- <li>REST server status (<code>ON</code> if server is started and detected, <code>OFF</code> otherwise).</li>
- <li>Local probe status (<code>ON</code> if probe is started and connected to the server, <code>OFF</code> otherwise).</li>
- <li>REST access token if the user is signed in. This token is required for all REST calls.</li>
- <li>Current user working directory.</li>
- </ul>
- <p>
- Note that when using <code>start-server</code> command the <a href="/using-rest.html#users">default user account</a> is automatically signed in
- upon the server start and the access token is obtained. You can sign in with different user account, if necessary.
- </p>
- </div>
- <h2 class="section-sub-title">Get Command List <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Type <code>help</code> to list all supported commands (or run <code>bin/nlpcraft.{sh|cmd} help</code> if in command line mode):
- </p>
- <p>
- <img class="img-fluid" src="/images/cli2.png" alt="">
- </p>
- <p>
- You can get detailed instructions and usage examples for any command
- <code>xxx</code> by typing <code>help --cmd=xxx</code>, for example:
- </p>
- <p>
- <img style="max-width: 914px !important;" class="img-fluid" alt="" src="/images/cli7.png">
- </p>
- <p>
- Use <span class="keyboard">Tab</span> key anytime for auto-completion for commands, parameters, file system paths, and model class names.
- Use <span class="keyboard">↑</span> and <span class="keyboard">↓</span> keys to scroll through command history.
- </p>
- <p>
- To exit <code>bin/nlpcraft.{sh|cmd}</code> script in REPL mode use <code>quit</code> command.
- </p>
- </section>
- <section id="os_commands">
- <h2 class="section-title">OS Commands <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can run any external OS-specific commands from <code>bin/nlpcraft.{sh|cmd}</code> script by prepending such
- command with a <code>$</code> symbol. For example:
- </p>
- <pre class="brush: bash">
- > $git pull # Runs 'git pull' in the current directory.
- > $vi /home/myproject/pom.xml # Edit 'pom.xml' using vi.
- > $emacs /home/myproject/MyModel.scala # Edit 'MyModel.scala' using emacs.
- > $ls -la # Run 'ls-la' Unix command.
- > $cmd /c dir # Runs Windows 'dir' command in a separate shell.
- </pre>
- <p>
- Note that <em>stdin</em>, <em>stderr</em> and <em>stdout</em> of the new process will inherit from running <code>bin/nlpcraft.{sh|cmd}</code> script process.
- Running OS commands makes the most sense in REPL mode where you don't want to lose a session context while
- executing external OS commands. Technically, however, external OS commands can be executed from command line mode
- as well.
- </p>
- </section>
- <section id="tips">
- <h2 class="section-title">Tips <span class="amp">&</span> Tricks <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft CLI is standard command line utility that can be used in the familiar ways both in
- Linux/Unix/MacOS and Window environments.
- </p>
- <span class="section-sub-title">Typical Development Workflow <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Here's the typical workflow of working with NLPCraft using NLPCraft CLI. Note that although all of these
- steps can be performed from IDE or a command line - NLPCraft CLI in REPL model provides the easiest
- way to accomplish these tasks:
- </p>
- <ol>
- <li>
- <code>gen-project</code> command generates new Maven/Gradle/Sbt project for either Java, Scala, Groovy
- or Kotlin that is ready to run.
- </li>
- <li>
- <code>start-server</code> command starts the local REST server. You only need to do it once as it is a
- "fire and forget" component.
- </li>
- <li>
- Modify <span class="amp">&</span> build the project using IDE or editor of your choice. You can
- <a href="#os_commands">launch</a> the editor and build tooling right from NLPCraft CLI.
- </li>
- <li>
- <code>test-model</code> command runs auto-validator for your model automatically starting and stopping an
- embedded data probe. Alternatively, <code>start-probe</code> and <code>ask</code> commands allow to manually
- start the local data probe and issue REST requests.
- <ul>
- <li>
- Note that you can use <code>retest-model</code> and <code>restart-probe</code> commands in REPL mode
- to rerun the latest model test or restart the probe with the last set of parameters accordingly avoiding
- the retyping of all required parameters.
- </li>
- </ul>
- </li>
- <li>
- Repeat steps 3 and 4 as you develop, test and debug your models and applications.
- </li>
- <li>
- Once finished - <code>stop</code> command stops both local REST server and data probe. Note that stopping
- the server once again is an optional step as it is a "fire and forget" component and it can be reused between development
- sessions.
- </li>
- </ol>
- <div class="bq info">
- <p>
- Steps 3 and 4 define a typical development cycle for change, build and test. In most cases you <b>don't need</b>
- to restart the server between multiple sessions.
- </p>
- </div>
- <span class="section-sub-title">Using <code>start-server</code> Command <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- REST server is a fire-and-forget component in NLPCraft. You can start it once and you only need to restart it
- if you change the server configuration. It is customary to add <code>bin/nlpcraft.{sh|cmd} start-server</code>
- call to the OS init script to have it started automatically on boot up.
- </p>
- <span class="section-sub-title">Using Default <code>nlpcraft.conf</code> <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- It is recommended to modify the standard <code>nlpcraft.conf</code> configuration file, i.e. <em>in a standard
- installation location</em>. Commands like <code>start-server</code>, <code>start-probe</code> and
- <code>test-model</code> use this location by default and you will not have to specify alternative
- location or provide other overrides every time you run these commands.
- If you keep all your configuration modifications in the standard <code>nlpcraft.conf</code> file you can simply
- run these commands without any additional arguments - significantly simplifying their usage.
- </p>
- <span class="section-sub-title">Using NLPCraft CLI In Scripts <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- NLPCraft CLI can be used in your own scripts to automate your own specific workflow and tasks.
- All of the commands that utilize REST API produce JSON responses, for a example:
- </p>
- <p>
- <img class="img-fluid" src="/images/cli9.png" alt="">
- </p>
- <p>
- You can parse these JSON responses for your own needs in your scripts. For example, here's the
- code snippet to get the access token from the <code>signin</code> command (on Linux):
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh no-ansi no-logo signin --email=admin@admin.com --passwd=admin | tail -n +2 | jq -M '.acsTok' | tr -d '"'
- </pre>
- <p>
- ...and the access token is:
- </p>
- <pre class="brush: bash">
- bjok7yraypseyk86KgGae
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- We use <code>no-ansi</code> and <code>no-logo</code> commands that clean up and simplify the
- output so that it would be easier to parse the resulting JSON.
- </li>
- <li>
- We use <code>tail -n +2</code> Unix command to skip the 1st line of the output that contains HTTP return code.
- </li>
- <li>
- We use <a target=_blank href="https://stedolan.github.io/jq/">jq</a> utility to parse JSON inline and extract
- <code>acsTok</code> field.
- </li>
- <li>
- We use <code>tr -d '"'</code> to remove surrounding double quotes to get our final result.
- </li>
- </ul>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#quick_start">Quick Start</a></li>
- <li><a href="#os_commands">OS Commands</a></li>
- <li><a href="#tips">Tips <span class="amp">&</span> Tricks</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/sql_model_gen.html b/tools/sql_model_gen.html
deleted file mode 100644
index 6fa3bc8..0000000
--- a/tools/sql_model_gen.html
+++ /dev/null
@@ -1,272 +0,0 @@
----
-active_crumb: SQL Model Generator
-layout: documentation
-id: sql_model_gen
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When using NLPCraft to develop natural language interface to the existing SQL RDBMS one of the
- routine tasks is to develop a data model that mirrors the SQL schema for the given
- database. Doing it manually can be time consuming and error prone.
- </p>
- <p>
- SQL model generator automates this task. This is a Java-based utility that takes JDBC configuration,
- reads database schema using it and creates initial JSON or YAML stub for the data model. This stub then
- can be further extended and modified.
- </p>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This utility can be run in several ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-sql --url=jdbc:postgresql://localhost:5432/mydb --driver=org.postgresql.Driver --schema=public --out=model.json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-sql</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-class" role="tabpanel">
- <pre class="brush: bash">
- $ java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run this class without arguments to get a full help.
- </li>
- <li>
- Use <code>org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator</code> class directly to execute
- it from IDE or programmatically.
- </li>
- </ul>
- </div>
- </div>
- <h2 class="section-sub-title">Parameters <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- If you run this tool with <code>--help</code> parameter you'll get the full help information for each
- parameter:
- </p>
- <pre class="brush: text">
-NAME:
- NCSqlModelGenerator -- NLPCraft model generator from SQL databases.
-
-SYNOPSIS:
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator [PARAMETERS]
-
-DESCRIPTION:
- This utility generates NLPCraft model stub from a given SQL database schema. You
- can choose database schema, set of tables and columns for which you
- want to generate NLPCraft model. After the model is generated you can
- further configure and customize it for your specific needs.
-
- This Java class can be run from the command line or from an IDE like any other
- Java application. Note that required JDBC driver class must be available on the
- classpath and therefore its JAR should be added to the classpath when running
- this application.
-
-PARAMETERS:
- --url|-r=url
- Mandatory database JDBC URL.
-
- --driver|-d=class
- Mandatory JDBC driver class. Note that 'class' must be a
- fully qualified class name. It should also be available on
- the classpath.
-
- --schema|-s=schema
- Mandatory database schema to scan.
-
- --out|-o=filename
- Mandatory name of the output JSON or YAML model file. It should
- have one of the following extensions: .js, .json, .yml, or .yaml
- File extension determines the output file format.
-
- --user|-u=username
- Optional database user name.
-
- --password|-w=password
- Optional database user password.
-
- --mdlId|-m=id
- Optional generated model ID. By default, the model ID will be 'sql.model.id'.
-
- --mdlVer|-v=version
- Optional generated model version. By default, the model version will be '1.0.0-timestamp'.
-
- --mdlName|-n=name
- Optional generated model name. By default, the model name will be 'SQL-based model'.
-
- --exclude|-e=list
- Optional semicolon-separate list of tables and/or columns to exclude. By
- default, none of the tables and columns in the schema are excluded. See below
- for more information.
-
- --prefix|-f=list
- Optional comma-separate list of table or column name prefixes to remove.
- These prefixes will be removed when name is used for model elements
- synonyms. By default, no prefixes will be removed.
-
- --suffix|-q=list
- Optional comma-separate list of table or column name suffixes to remove.
- These suffixes will be removed when name is used for model elements
- synonyms. By default, no suffixes will be removed.
-
- --include|-i=list
- Optional semicolon-separate list of tables and/or columns to include. By
- default, all tables and columns in the schema are included. See below
- for more information.
-
- --synonyms|-y=true|false
- Optional flag on whether to generated auto synonyms for the model elements.
- Default is true.
-
- --override|-z=true|false
- Optional flag to determine whether to override output file if it already exist.
- If override is disabled (default) and output file exists - a unique file name will
- be used instead.
- Default is false.
-
- --parent|-p=true|false
- Optional flag on whether to use element's parent relationship for
- defining SQL columns and their containing (i.e. parent) tables.
- Default is false.
-
- --help|-h|-?
- Prints this usage information.
-
-DETAILS:
- -r, -d, -s, and -o are mandatory parameters, everything else is optional.
-
- Parameter values can be placed in double (") or single (') quotes which will be
- automatically discarded. Use it to pass strings containing spaces in the command line.
-
- Each -i or -e parameter is a semicolon (;) separated list of table or columns names.
- Each table or column name can be one of following forms:
- - table -- to filter on table names only.
- - table#column -- to filter on both table and column names.
- - #column -- to filter on columns only (regardless of the table).
-
- Table and column names are treated as standard Java regular expressions. Note that
- both '#' and ';' cannot be used inside of the regular expression:
-
- -e="#_.+" -- excludes any columns starting with '_'.
- -e="tmp.+" -- excludes all tables starting with 'tmp'.
- -i="Order.*;#[^_].+" -- includes only tables starting with 'Order' and columns that
- do not start with '_'.
-
-EXAMPLES:
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator
- -r=jdbc:postgresql://localhost:5432/mydb
- -d=org.postgresql.Driver
- -f="tbl_, col_"
- -q="_tmp, _old, _unused"
- -s=public
- -e="#_.+"
- -o=model.json
- </pre>
- <p>
- After the data model stub is generated:
- </p>
- <ul>
- <li>
- Load generated YAML/JSON-based model using <a target="javadoc" href="https://javadoc.io/static/org.apache.nlpcraft/nlpcraft/0.5.0/org/apache/nlpcraft/model/NCModelFileAdapter.html">NCModelFileAdapter</a>
- class to instantiate model from this file.
- </li>
- <li>
- Modify and extend generated model stub to your own needs. In most cases, you'll need
- to add, remove or modify auto-generated synonyms, add intents, etc. Note, however, that generated model
- is valid and minimally complete and can be used as is.
- </li>
- <li>
- Use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractorBuilder.html">NCSqlSchemaBuilder</a> class to get an object representation of the
- SQL data schema for the model. You can use this object representation along with many utility
- methods in <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractor.html">NCSqlExtractor</a>
- class to efficiently auto-generate SQL queries against the source RDBMS.
- </li>
- </ul>
- <p>
- SQL model generator comes with a several utility classes that can be used to programmatically explore
- generated data model, its elements and their metadata. You should start with the following builders to learn more
- about these utility classes:
- </p>
- <ul>
- <li>
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractorBuilder.html">NCSqlExtractorBuilder</a>
- </li>
- <li>
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlSchemaBuilder.html">NCSqlSchemaBuilder</a>
- </li>
- </ul>
- </section>
- <section id="examples">
- <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- SQL model generator is used by <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">SQL Model</a> example. This example comes with two YAML models:
- </p>
- <ul>
- <li><code>sql_model_init.yaml</code> - original generated model straight out of the SQL model generator.</li>
- <li><code>sql_model.yaml</code> - modified and updated model ultimately used by the example.</li>
- </ul>
- <p>
- It is recommended to run a diff between these two files to see what was actually changed and how.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- <li><a href="#examples">Examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/syn_tool.html b/tools/syn_tool.html
deleted file mode 100644
index 2be6fea..0000000
--- a/tools/syn_tool.html
+++ /dev/null
@@ -1,314 +0,0 @@
----
-active_crumb: Synonyms Tool
-layout: documentation
-id: syn_tool
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Synonym suggester tool takes an existing model, analyses its synonyms and intents and comes up with
- a list of synonyms that are currently missing that you might want to add to your model.
- </p>
- <p>
- This tool is accessed via REST call. It is based on Google's BERT and Facebook fasttext
- models. It requires <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> or
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations present on intent
- callbacks. When invoked, the tool scans the given data model for intents and these annotations, and based on these samples tries to determine
- which synonyms are missing in the model.
- </p>
- <div class="bq info">
- <p>
- <b>Single Word Synonyms</b>
- </p>
- <p>
- Synonym suggester tool analyses only single word synonyms ignoring any multi-word synonyms. You
- can often convert a named element with multi-word synonyms into a combination of multiple named
- elements each with a single word synonyms using <a href="/data-model.html#dsl">Composable NERs</a>
- technique.
- </p>
- </div>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In order to use this tool the <code>ctxword</code> server and NLPCraft server should be started as well as
- the server's configuration should potentially be updated.
- </p>
- <h2 class="section-sub-title"><code>ctxword</code> Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <div class="bq warn">
- <b>Python 3.6-3.8</b>
- <p>
- As of this writing (Dec 2020) the <code>ctxword</code> server and its dependencies work only with Python 3.6-3.8 version.
- </p>
- </div>
- <p>
- 'ctxword' server is a Python-based module that provides BERT and fasttext based implementation
- for finding a contextually related words for a given word from the input sentence. NLPCraft server interacts
- with 'ctxword' server via internal REST interface. To configure NLPCraft server and start 'ctxword' Python-based
- server follow these steps:
- </p>
- <ol>
- <li>
- <p>
- Install necessary dependencies by running the following commands from the NLPCraft installation
- directory:<br/>
- <b>NOTE:</b> this step should only be performed once.
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-nix" role="tab">Linux/Unix/MacOS</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-win" role="tab">Windows</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-nix" role="tabpanel">
- <pre class="brush: bash">
- $ cd nlpcraft/src/main/python/ctxword
- $ bin/install_dependencies.sh
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-win" role="tabpanel">
- <p></p>
- <p>
- Read <code>src\main\python\ctxword\bin\WINDOWS_SETUP.md</code> file for manual installation instructions.
- </p>
- </div>
- </div>
- </li>
- <li>
- <em>Optional.</em>
- <br/>
- Configure <code>nlpcraft.server.ctxword.url</code> property in <code>nlpcraft.conf</code> file (or your own configuration file).
- This property comes with a default endpoint and you only need to change it if you change the
- 'ctxword' module implementation.
- </li>
- <li>
- Start the 'ctxword' server by running the following commands from NLPCraft installation directory:
- <pre class="brush: bash">
- $ cd nlpcraft/src/main/python/ctxword
- $ bin/start_server.{sh|cmd}
- </pre>
- <div class="bq info">
- <p>
- <b>1st Start</b>
- </p>
- Note that on the first start the server will try to load compressed BERT model which is not yet
- available. It will then download this library and compress it which will take a several minutes
- and may require 10 GB+ of available memory. Subsequent starts will skip this step, and the
- server will start much faster.
- </div>
- </li>
- </ol>
- <h2 class="section-sub-title">REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server should be <a href="/server-and-probe.html#server">started</a>.
- </p>
- <h2 class="section-sub-title">Running <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Synonyms tool can be run in two different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-rest" role="tab">REST Call</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh help --cmd=model-sugsyn
- $ bin/nlpcraft.sh model-sugsyn --mdlId=nlpcraft.alarm.ex --minScore=0.5
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <code>mldId</code> parameter is only required if there is more than one model
- deployed in the connected data probe. If the data probe has only one model you can
- ommit this parameter.
- </li>
- <li>
- <code>minScore</code> - Optional minimum confidence score to include into the result, ranging from 0 to 1, default is 0.
- <code>minScore</code> of 0 will include all results, and <code>minScore</code> of 1 will include only results
- with the absolutely highest confidence score. Values between 0.5 and 0.7 is generally suggested.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=model-sugsyn</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-rest" role="tabpanel">
- <p></p>
- <p>
- <a href="/using-rest.html">REST API</a> accepts only <code>POST</code> HTTP calls and <code>application/json</code> content type
- for JSON payload and responses. When issuing a REST call for this tool you will be using the following URL:
- </p>
- <pre class="brush: plain">
- https://localhost:8081/api/v1/model/sugsyn
- </pre>
- <p>
- where:
- <dl>
- <dt><code>http</code></dt>
- <dd>Either <code>http</code> or <code>https</code> protocol.</dd>
- <dt><code>localhost:8081</code></dt>
- <dd>Host and port on which REST server is started. <code>localhost:8081</code> is the default configuration and can be <a href="/server-and-probe.html">changed</a>.</dd>
- <dt><code>/api/v1</code></dt>
- <dd>Mandatory prefix indicating API version.</dd>
- <dt><code>model/sugsyn</code></dt>
- <dd>Synonym suggester REST call.</dd>
- </dl>
- <p>
- The parameters should be passed in as JSON:
- </p>
- <pre class="brush: js">
- {
- "acsTok": "qweqw9123uqwe",
- "mdlId": "nlpcraft.alarm.ex",
- "minScore": 0.5
- }
- </pre>
- <p>
- where:
- </p>
- <ul>
- <li>
- <code>acsTok</code> - access token obtain via previous <code>'/signin'</code> call.
- </li>
- <li>
- <code>mdlId</code> - ID of the model to run synonym suggester on.
- </li>
- <li>
- <code>minScore</code> - Optional minimum confidence score to include into the result, ranging from 0 to 1, default is 0.
- <code>minScore</code> of 0 will include all results, and <code>minScore</code> of 1 will include only results
- with the absolutely highest confidence score. Values between 0.5 and 0.7 is generally suggested.
- </li>
- </ul>
- </div>
- </div>
- <p>
- Either way the synonym suggester returns the following JSON result (<code>nlpcraft.alarm.ex</code>
- model from <a href="/examples/alarm_clock.html">Alarm</a> example):
- </p>
- <pre class="brush: js">
-{
-"status": "API_OK",
-"result": {
- "modelId": "nlpcraft.alarm.ex",
- "minScore": 0.5,
- "durationMs": 424.0,
- "timestamp": 1.60091239852E12,
- "suggestions": [
- {
- "x:alarm": [
- {
- "score": 1.0,
- "synonym": "ask"
- },
- {
- "score": 0.9477103542042674,
- "synonym": "join"
- },
- {
- "score": 0.8882341083867801,
- "synonym": "get"
- },
- {
- "score": 0.7330826349218547,
- "synonym": "remember"
- },
- {
- "score": 0.6902880910527778,
- "synonym": "contact"
- },
- {
- "score": 0.6014764219771813,
- "synonym": "time"
- },
- {
- "score": 0.5816398376889104,
- "synonym": "follow"
- },
- {
- "score": 0.5640882890681899,
- "synonym": "watch"
- },
- {
- "score": 0.5139855649326083,
- "synonym": "stop"
- },
- {
- "score": 0.5136895804732818,
- "synonym": "kill"
- },
- {
- "score": 0.5001167992233122,
- "synonym": "send"
- }
- ]
- }
- ],
- "warnings": [
- "Model has too few (3) intents samples. It will negatively affect the quality of suggestions. Try to increase overall sample count to at least 20."
- ]
-}
- </pre>
- <p>
- The result is structured as a list of proposed synonyms with their corresponding scores for each model's
- element.
- You should analyse the results for their fitness for your model and its existing synonyms. The tool cannot guarantee
- that every suggested synonym is appropriate or valid - but it gives a good "courtesy" check for potentially
- missing synonyms.
- </p>
- <div class="bq info">
- <p>
- <b>Run Periodically</b>
- </p>
- <p>
- It is a good idea to run this tool periodically if you are actively changing the model. With dozens or hundreds
- of model elements it is very hard to manually maintain quality set of synonyms. With a good list of
- user input samples for each intent this tool can be indispensable for easy maintenance of the synonyms.
- </p>
- </div>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/test_framework.html b/tools/test_framework.html
deleted file mode 100644
index 72d0d4f..0000000
--- a/tools/test_framework.html
+++ /dev/null
@@ -1,324 +0,0 @@
----
-active_crumb: Test Framework
-layout: documentation
-id: test_framework
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column" xmlns="http://www.w3.org/1999/html">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft comes built-in with a simple-to-use testing framework that can be used with any
- popular unit testing JVM frameworks such as <a href="https://testng.org">TestNG</a> or
- <a href="https://junit.org">JUnit</a>. Test framework is essentially a simplified implementation
- of the REST client tailor-made for model unit testing.
- </p>
- <p>
- Test framework consists of the following types in <code>org.apache.nlpcraft.model.tools.test</code> package:
- </p>
- <ul>
- <li>
- Auto-testing:
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">
- NCTestAutoModelValidator
- </a> - auto-runner for model tests based on <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- and <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations.
- </li>
- </ul>
- </li>
- <li>
- Using test client directly:
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClient.html">
- NCTestClient
- </a> - main model test client interface.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClientBuilder.html">
- NCTestClientBuilder
- </a> - builder for <code>NCTestClient</code> instances.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestResult.html">
- NCTestResult
- </a> - result holder of the test sentence processing.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClientException.html">
- NCTestClientException
- </a> - exception used by the test framework.
- </li>
-
- </ul>
- </ul>
- </section>
- <section id="usage">
- <h2 class="section-title">Test client <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Here's a code for unit test using test client for the model from <a href="/examples/alarm_clock.html">Alarm Clock</a>
- example illustrating the usage of test framework together with JUnit 5:
- </p>
- <pre class="brush: java, highlight: [8, 10, 16, 24, 25, 26]">
-public class AlarmTest {
- private NCTestClient cli;
-
- @BeforeEach
- void setUp() throws NCException, IOException {
- NCEmbeddedProbe.start(AlarmModel.class);
-
- cli = new NCTestClientBuilder().newBuilder().build();
-
- cli.open("nlpcraft.alarm.ex");
- }
-
- @AfterEach
- void tearDown() throws NCException, IOException {
- if (cli != null)
- cli.close();
-
- NCEmbeddedProbe.stop();
- }
-
- @Test
- public void test() throws NCException, IOException {
- // Should be passed.
- assertTrue(cli.ask("Ping me in 3 minutes").isOk());
- assertTrue(cli.ask("Buzz me in an hour and 15mins").isOk());
- assertTrue(cli.ask("Set my alarm for 30s").isOk());
- }
-}
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Before each unit test execution on line 8 we create new instance of <code>NCTestClient</code> using
- client builder with default configuration. On line 10 we open test client connection for model
- with ID <code>nlpcraft.alarm.ex</code>.
- </li>
- <li>
- After each test execution we close test client connection on line 16.
- </li>
- <li>
- On lines 24-26 we submit ("ask") test sentences to our model and check for succesfull processing.
- </li>
- </ul>
- <div class="bq info">
- <p>
- <b>Embedded Probe</b>
- </p>
- <p>
- Note that this example (lines 6 and 18), as all other examples shipped with NLPCraft, uses
- <a href="embedded_probe.html">Embedded Probe</a>.
- </p>
- <p>
- Typically, data probes are launched in their own independent JVMs. However, in some
- cases - e.g. during unit testing - it is more convenient for model implementation or preferable
- for performance reasons to host a data model (and hence the data probe) in the same JVM.
- </p>
- </div>
- </section>
- <section id="autotest">
- <h2 class="section-title">Auto Model Validation <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- The same model from <a href="/examples/alarm_clock.html">Alarm Clock</a> example can be auto validated using
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a>
- class and user defined <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations from the model's callback method:
- </p>
- <pre class="brush: java, highlight: [4, 5, 6]">
-public class AlarmModel extends NCModelFileAdapter {
- @NCIntentRef("alarm")
- @NCIntentSample({
- "Ping me in 3 minutes",
- "Buzz me in an hour and 15mins",
- "Set my alarm for 30s"
- })
- NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("nums") List<NCToken> numToks
- ) {
- // ...
- }
-}
- </pre>
- <p>
- Auto model validator takes one or more model IDs (or class names) and performs validation. Validation consists
- of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model, scanning for
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations and their corresponding callback methods, submitting each
- sample input sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- </p>
- <h2 class="section-sub-title">Running <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Auto mode validator can be executed in two different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh help --cmd=test-model
- $ bin/nlpcraft.sh test-model --cp=/path/to/my/model/classes
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- In REPL mode you can use <code>retest-model</code> command to re-run the last mode test with the
- same parameters avoiding the need to retype all the parameters again.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-class" role="tabpanel">
- <pre class="brush: bash">
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar:/path/to/my/model/classes -DNLPCRAFT_TEST_MODELS=org.apache.nlpcraft.examples.alarm.AlarmModel org.apache.nlpcraft.model.tools.test.NCTestAutoModelValidator
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Make sure to add necessary classpath components for the model(s) you want to auto-validate.
- </li>
- <li>
- Specify the following three system property (all optional):
- <table class="gradient-table">
- <thead>
- <tr>
- <th>System Property</th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code>NLPCRAFT_TEST_MODELS</code></td>
- <td>
- <p>
- Optional system property that should contain comma separate
- list of the data model classes to test. If not provided, the models from the probe
- configuration will be used.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-DNLPCRAFT_TEST_MODELS=org.apache.nlpcraft.examples.alarm.AlarmModel</code>
- </p>
- </td>
- </tr>
- <tr>
- <td><code>NLPCRAFT_PROBE_CONFIG</code></td>
- <td>
- <p>
- Optional system property that should contain the path to probe configuration file.
- If not provided, the default probe configuration file will be used.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-NLPCRAFT_PROBE_CONFIG=/opt/my_probe/probe.conf</code>
- </p>
- </td>
- </tr>
- <tr>
- <td><code>NLPCRAFT_TEST_INTENT_IDS</code></td>
- <td>
- <p>
- Optional system property that should contain the comma separated list of
- intent IDs to test. If not provided, all detected intents will be tested.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-NLPCRAFT_TEST_INTENT_IDS=int1,int2</code>
- </p>
- </td>
- </tr>
- </tbody>
- </table>
- </li>
- <li>
- You can also use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> class
- directly to call it programmatically from the code or from IDE.
- </li>
- </ul>
- </div>
- </div>
- <p>
- In the log output you should see the following validation results:
- </p>
- <figure>
- <img class="img-fluid-no-border" src="/images/auto_validation.png" alt="">
- <figcaption><b>Fig 1.</b> Model auto-validation result.</figcaption>
- </figure>
- <p>
- See <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a>
- class for more details.
- </p>
- </section>
- <section id="examples">
- <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- All <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">examples</a>
- shipped with NLPCraft come with instructions on how to auto-validate their models.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Test client</a></li>
- <li><a href="#autotest">Auto model validation</a></li>
- <li><a href="#examples">Examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/using-rest.html b/using-rest.html
deleted file mode 100644
index 78890a2..0000000
--- a/using-rest.html
+++ /dev/null
@@ -1,232 +0,0 @@
----
-active_crumb: REST API
-layout: documentation
-id: rest
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--suppress HtmlDeprecatedTag -->
-<div class="col-md-8 second-column">
- <section>
- <h2 class="section-title">TL;DR; <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- View <a target="swaggerhub" href="https://app.swaggerhub.com/apis-docs/Apache-NLPCraft/apache-nlpcraft/{{site.latest_version}}">Open API Specification</a> in a separate window. 👌
- </p>
- </section>
- <section id="overview">
- <h2 class="section-title">...bit more details <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST API provides a universal integration path for any type of user application to connect to and use NLPCraft.
- By using REST API these applications can ask natural language questions, get results and perform
- many supporting operations such as user management.
- </p>
- <p>
- REST server accepts the REST call,
- interprets and routes it to the appropriate data probe which in turn invokes one of its deployed
- data models. The result of data model processing travels all the way through the same chain of components
- back to the user application that made the original call.
- </p>
- <div class="bq info">
- <p>
- If you downloaded ZIP archive the <a target="swaggerhub" href="https://app.swaggerhub.com/apis-docs/Apache-NLPCraft/apache-nlpcraft/{{site.latest_version}}">REST API specification</a> is also available in
- <code>openapi/nlpcraft_swagger.yml</code> file
- or on <a target="github" href="https://github.com/apache/incubator-nlpcraft/blob/master/openapi/nlpcraft_swagger.yml">GitHub</a>.
- </p>
- </div>
- </section>
- <section id="java-client">
- <h2 class="section-title">Java Client <i class="fab fa-java"></i> <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft provide native Java client that enables easy-to-use Java-based API wrapping standard
- NLPCraft REST APIs. It can be used by any JVM language that provides Java interop such as Scala, Groovy, or Kotlin:
- </p>
- <ul>
- <li>Maven/Grape/Gradle/SBT <a href="download.html#java-client">instructions</a></li>
- <li>Latest <a target=_ href="/apis/java-client/latest/index.html">Javadoc</a></li>
- <li>Watch, star or fork <a target="github" href="https://github.com/apache/incubator-nlpcraft-java-client">GitHub Project</a></li>
- </ul>
-
- </section>
- <section id="format">
- <h2 class="section-title">REST URL <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST API accepts only <code>POST</code> HTTP calls and <code>application/json</code> content type
- for JSON payload and responses. When issuing a REST call you will be using the following URL:
- </p>
- <pre class="brush: plain">
- https://localhost:8081/api/v1/signin
- </pre>
- <p>
- where:
- <dl>
- <dt><code>http</code></dt>
- <dd>Either <code>http</code> or <code>https</code> protocol.</dd>
- <dt><code>localhost:8081</code></dt>
- <dd>Host and port on which REST server is started. <code>localhost:8081</code> is the default configuration and can be <a href="/server-and-probe.html">changed</a>.</dd>
- <dt><code>/api/v1</code></dt>
- <dd>Mandatory prefix indicating API version.</dd>
- <dt><code>/signin</code></dt>
- <dd>Specific REST path.</dd>
- </dl>
- <div class="bq info">
- <p>
- <b>Management Script</b>
- </p>
- <p>
- <a href="/tools/script.html"><code>bin/nlpcraft.{sh|cmd}</code></a> script is a <b>central management utility</b> for NLPCraft. Among many
- commands it provides, there are two commands <code>call</code> and <code>rest</code> that allows for simple and
- effective REST API usage against local REST server.
- </p>
- </div>
- </section>
- <section id="users">
- <h2 class="section-title">Users <span class="amp">&</span> Organizations <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In NLPCraft most of the REST calls require ID of the user. Every user belongs to a company, and each user
- has administrative privileges flag. The notion of the user and company IDs additionally to the model ID provide
- necessary attributes for proper probe authentication, conversation maintenance, and security.
- </p>
- <h2 class="section-sub-title">Default Account <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft comes with the default account:
- </p>
- <ul>
- <li><b>Email: </b>admin@admin.com</li>
- <li><b>Password: </b>admin</li>
- </ul>
- <p>
- This account has administrative privileges and should be used as a bootstrap account to create other accounts.
- Make sure to delete this account when you go to production.
- </p>
- <div class="bq warn">
- <p><b><span style="font-size: 120%; color: #E67E22">⚠</span> Default Account <span style="font-size: 120%; color: #E67E22">⚠</span></b></p>
- <p>
- Make sure to delete the default account when you go to production or as soon as you have other secure
- administrative account that can act as a bootstrap account.
- </p>
- </div>
- <h2 class="section-sub-title">Admins <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many operations are only available to the users with administrative privileges. Note
- also that some operations will implicitly behave differently based on whether the currently signed in user
- have administrative privileges or not.
- </p>
- <h2 class="section-sub-title">External User ID <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many calls on REST API accept external "on-behalf-of" user ID (<code>usrExtId</code>) additionally to the regular
- user ID (<code>usrID</code>). In these methods zero, one or both IDs must be provided. If none are provided
- the ID of the currently signed in user will be used. If both are provided they should point to the same user.
- External user ID allows to use user identification from the external systems without a need to import all the
- existing users into NLPCraft in the first place.
- </p>
- <p>
- Typical usage pattern for integrating NLPCraft into existing
- system is to create a single administrative NLPCraft user, sign in with that account and then use external
- "on-behalf-of" user IDs in all the requests. This way you get the same functionality as if using ordinary
- user IDs but without a need to migrate and synchronize all user accounts from the existing system to NLPCraft.
- </p>
- </section>
- <section id="server_errors">
- <h2 class="section-title">Server Errors <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When REST server returns HTTP error the response body contains JSON object (<code>Content-Type → application/json</code>)
- with two fields <code>code</code> and <code>msg</code>:
- </p>
- <pre class="brush: js">
- {
- "code": "NC_INVALID_ACCESS_TOKEN",
- "msg": "Unknown access token: PPdxjwXBOIMpAWNgpKq1"
- }
- </pre>
- <p>
- Following tables shows all possible <code>code</code> values for these server errors:
- </p>
- <table class="gradient-table">
- <thead>
- <tr>
- <th>Code</th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code>NC_INVALID_ACCESS_TOKEN</code></td>
- <td>
- The access token is invalid or no longer valid. Note that previously issued existing access
- tokens can expire or be otherwise invalidated and have to be obtained again.
- </td>
- </tr>
- <tr>
- <td><code>NC_SIGNIN_FAILURE</code></td>
- <td>
- Invalid or unknown user email and/or user password during <code>/signin</code> call. This can
- indicate either the attempt to signin by the user that hasn't been added yet, or the invalid
- password for the existing user.
- </td>
- </tr>
- <tr>
- <td><code>NC_NOT_IMPLEMENTED</code></td>
- <td>
- Particular feature is not implemented yet or not available. This is reserved for API extensions and
- future backward compatibility.
- </td>
- </tr>
- <tr>
- <td><code>NC_INVALID_FIELD</code></td>
- <td>
- This indicated a problem with a field. It's either too bit, too small, empty or otherwise invalid.
- See <code>msg</code> field in the error response for details.
- </td>
- </tr>
- <tr>
- <td><code>NC_ADMIN_REQUIRED</code></td>
- <td>
- Calling user is required to have admin privileges to execute given call.
- </td>
- </tr>
- <tr>
- <td><code>NC_INVALID_OPERATION</code></td>
- <td>
- Indicates that a given operation cannot be performed as specified. For example, this error
- is returned when you attempt to remove or revoke admin privileges from the
- last admin user.
- </td>
- </tr>
- <tr>
- <td><code>NC_ERROR</code></td>
- <td>Indicates a general server error.</td>
- </tr>
- </tbody>
- </table>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#java-client">Java Client</a></li>
- <li><a href="#format">REST URL</a></li>
- <li><a href="#users">Users <span class="amp">&</span> Org</a></li>
- <li><a href="#server_errors">Server Errors</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
