WIP.
diff --git a/_includes/left-side-menu.html b/_includes/left-side-menu.html
index 0ac7ba6..64e44c5 100644
--- a/_includes/left-side-menu.html
+++ b/_includes/left-side-menu.html
@@ -60,69 +60,12 @@
{% endif %}
</li>
<li>
- {% if page.id == "server_and_probe" %}
- <a class="active" href="/server-and-probe.html">Server <span class="amp">&</span> Probe</a>
- {% else %}
- <a href="/server-and-probe.html">Server <span class="amp">&</span> Probe</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "metrics" %}
- <a class="active" href="/metrics-and-tracing.html">Metrics <span class="amp">&</span> Tracing</a>
- {% else %}
- <a href="/metrics-and-tracing.html">Metrics <span class="amp">&</span> Tracing</a>
- {% endif %}
- </li>
- <li>
{% if page.id == "integrations" %}
<a class="active" href="/integrations.html">Integrations</a>
{% else %}
<a href="/integrations.html">Integrations</a>
{% endif %}
</li>
- <li>
- {% if page.id == "rest" %}
- <a class="active" href="/using-rest.html">REST API</a>
- {% else %}
- <a href="/using-rest.html">REST API</a>
- {% endif %}
- </li>
- <li class="side-nav-title">Tools</li>
- <li>
- {% if page.id == "script" %}
- <a class="active" href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a>
- {% else %}
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "test_framework" %}
- <a class="active" href="/tools/test_framework.html">Test Framework</a>
- {% else %}
- <a href="/tools/test_framework.html">Test Framework</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "embedded_probe" %}
- <a class="active" href="/tools/embedded_probe.html">Embedded Probe</a>
- {% else %}
- <a href="/tools/embedded_probe.html">Embedded Probe</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "sql_model_gen" %}
- <a class="active" href="/tools/sql_model_gen.html">SQL Model Generator</a>
- {% else %}
- <a href="/tools/sql_model_gen.html">SQL Model Generator</a>
- {% endif %}
- </li>
- <li>
- {% if page.id == "syn_tool" %}
- <a class="active" href="/tools/syn_tool.html">Synonyms Tool</a>
- {% else %}
- <a href="/tools/syn_tool.html">Synonyms Tool</a>
- {% endif %}
- </li>
<li class="side-nav-title">Examples</li>
<li>
{% if page.id == "calculator" %}
diff --git a/examples/alarm_clock.html b/examples/alarm_clock.html
deleted file mode 100644
index a85af49..0000000
--- a/examples/alarm_clock.html
+++ /dev/null
@@ -1,417 +0,0 @@
----
-active_crumb: Alarm Clock <code><sub>ex</sub></code>
-layout: documentation
-id: alarm_clock
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This simple example provides an "alarm clock" implementation where you can ask to set the timer
- for a specific duration from now expressed in hours, minutes and/or seconds. You can say "ping me in 3 minutes",
- "buzz me in an hour and 15 minutes", or "set my alarm for 30 secs". When the timer is up it will
- simply print out "BEEP BEEP BEEP" in the data probe console.
- </p>
- <p>
- <b>Complexity:</b> <span class="complexity-one-star"><i class="fas fa-star"></i> <i class="far fa-star"></i> <i class="far fa-star"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/alarm">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="new_project">
- <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can create new Java project in many ways - we'll use <a href="/tools/script.html">NLPCraft CLI</a>
- to accomplish this task:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-prj-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-prj-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-prj-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=AlarmClock --outputDir=~ --pkgName=demo --mdlType=json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/AlarmClock</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Java and Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-prj-out" role="tabpanel">
- <p></p>
- <img alt="" class="img-fluid" src="/images/alarm_clock_fig1.png">
- </div>
- </div>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to start with declaring the static part of our model using JSON which we will later load using
- <code>NCModelFileAdapter</code> in our Java-based model implementation. Open <code>src/main/resources/<b>alarm_clock.json</b></code>
- file and replace its content with this JSON:
- </p>
- <pre class="brush: js, highlight: [7, 11, 20]">
-{
- "id": "nlpcraft.alarm.ex",
- "name": "Alarm Example Model",
- "version": "1.0",
- "description": "Alarm example model.",
- "enabledBuiltInTokens": [
- "nlpcraft:num"
- ],
- "elements": [
- {
- "id": "x:alarm",
- "description": "Alarm token indicator.",
- "synonyms": [
- "{ping|buzz|wake|call|hit} {me|up|me up|_}",
- "{set|_} {my|_} {wake|wake up|_} {alarm|timer|clock|buzzer|call} {up|_}"
- ]
- }
- ],
- "intents": [
- "intent=alarm term~{# == 'x:alarm'} term(nums)~{# == 'nlpcraft:num' && meta_token('nlpcraft:num:unittype') == 'datetime' && meta_token('nlpcraft:num:isequalcondition') == true}[0,7]"
- ]
-}
- </pre>
- <p>There are number of important points here:</p>
- <ul>
- <li>
- <code>Line 7</code> indicates that we'll be using built-in token <code>nlpcraft:num</code> (and therefore
- it needs to be explicitly enabled).
- </li>
- <li>
- <code>Line 11</code> defines the only custom model element we'll need. It's ID is <code>x:alarm</code> and
- it is defined through synonyms. It basically means a verb to set up an alarm clock.
- </li>
- <li>
- On <code>line 20</code> we define an intent <code>alarm</code> that we are going to be looking for in the user input that
- consists of two terms: one for <code>x:alarm</code> element (defined above) and another one for up to 7 numeric values
- of unit type <code>datetime</code> (minutes, seconds, hours, etc.).
- </li>
- </ul>
- <p>
- Now that our model is ready let's create a Java class that would load this model and provide the actual
- callback for when the intent <code>alarm</code> is detected in the user input.
- </p>
- </section>
- <section id="code">
- <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Open <code>src/main/java/demo/<b>AlarmClock.java</b></code> file and replace its content with the following code:
- </p>
- <pre class="brush: java, highlight: [10, 18, 22, 27], 82">
-package demo;
-
-import org.apache.nlpcraft.model.*;
-import java.time.*;
-import java.time.format.*;
-import java.util.*;
-
-import static java.time.temporal.ChronoUnit.*;
-
-public class AlarmClock extends NCModelFileAdapter {
- private static final DateTimeFormatter FMT =
- DateTimeFormatter.ofPattern("HH'h' mm'm' ss's'").withZone(ZoneId.systemDefault());
-
- private final Timer timer = new Timer();
-
- public AlarmClock() {
- // Loading the model from the file in the classpath.
- super("alarm_model.json");
- }
-
- @NCIntentRef("alarm")
- @NCIntentSample({
- "Ping me in 3 minutes",
- "Buzz me in an hour and 15mins",
- "Set my alarm for 30s"
- })
- private NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("nums") List<NCToken> numToks
- ) {
- if (ctx.isAmbiguous())
- throw new NCRejection("Not exact match.");
-
- long unitsCnt = numToks.stream().map(tok -> (String)tok.meta("num:unit")).distinct().count();
-
- if (unitsCnt != numToks.size())
- throw new NCRejection("Ambiguous time units.");
-
- LocalDateTime now = LocalDateTime.now();
- LocalDateTime dt = now;
-
- for (NCToken num : numToks) {
- String unit = num.meta("nlpcraft:num:unit");
-
- // Skip possible fractional to simplify.
- long v = ((Double)num.meta("nlpcraft:num:from")).longValue();
-
- if (v <= 0)
- throw new NCRejection("Value must be positive: " + unit);
-
- switch (unit) {
- case "second": { dt = dt.plusSeconds(v); break; }
- case "minute": { dt = dt.plusMinutes(v); break; }
- case "hour": { dt = dt.plusHours(v); break; }
- case "day": { dt = dt.plusDays(v); break; }
- case "week": { dt = dt.plusWeeks(v); break; }
- case "month": { dt = dt.plusMonths(v); break; }
- case "year": { dt = dt.plusYears(v); break; }
-
- default:
- // It shouldn't be assert, because 'datetime' unit can be extended.
- throw new NCRejection("Unsupported time unit: " + unit);
- }
- }
-
- long ms = now.until(dt, MILLIS);
-
- assert ms >= 0;
-
- timer.schedule(
- new TimerTask() {
- @Override
- public void run() {
- System.out.println(
- "BEEP BEEP BEEP for: " + ctx.getContext().getRequest().getNormalizedText() + ""
- );
- }
- },
- ms
- );
-
- return NCResult.text("Timer set for: " + FMT.format(dt));
- }
-
- @Override
- public void onDiscard() {
- // Clean up when model gets discarded (e.g. during testing).
- timer.cancel();
- }
-}
- </pre>
- <p>
- There's a bit of a logic here that deals mostly with taking multiple numeric values and converting them into
- a single number of milliseconds that the alarm clock needs to be set up for. Let's review it step by step:
- </p>
- <ul>
- <li>
- On <code>line 10</code> our class extends <code>NCModelFileAdapter</code> that allows us to load most
- of the model declaration from the external JSON or YAML file (line 18) and only provide functionality that we
- couldn't express in declarative portion in JSON.
- </li>
- <li>
- <code>Line 27</code> defines method <code>onMatch</code> as a callback for intent <code>alarm</code>
- when it is detected in the user input. Method parameter <code>numToks</code> will get up to 7 tokens
- of type <code>nlpcraft:num</code> (see intent definition above).
- </li>
- <li>
- Note the <code>line 22</code> where we use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation to provide samples of the user input that this intent should match. Apart from documentation
- purpose these samples will be used when we will be <a href="#testing">testing out model below.</a>
- </li>
- <li>
- The rest of the method <code>onMatch</code> implementation is a relatively straight forward Java code
- that calculates timer delay from multiple numeric units and their types. In the end (line 68)
- it schedules a timer to print "BEEP BEEP BEEP" at calculated time. For simplicity, this message will
- be printed right in the data probe console.
- </li>
- <li>
- On the <code>line 82</code> the intent callback simply returns a confirmation message telling
- for what actual time the alarm clock was set.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Once we have our model ready let's go to <code>~/AlarmClock</code> directory and run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/AlarmClock
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component that you generally needs to start only once for this and any other
- examples.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Remember the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our model code next to intent definition?
- </p>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/AlarmClock/target/classes --mdls=demo.AlarmClock
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- <p>
- Look at the output of this command and you will see the test results for all our sample utterances:
- </p>
- <p>
- <img style="max-width: 667px !important;" class="img-fluid" alt="" src="/images/alarm-clock-test.png">
- </p>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created alarm clock data model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#new_project">New Project</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#code">Model Class</a></li>
- <li><a href="#build_project">Build Project</li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/examples/sql_model.html b/examples/sql_model.html
deleted file mode 100644
index 90eb96d..0000000
--- a/examples/sql_model.html
+++ /dev/null
@@ -1,300 +0,0 @@
----
-active_crumb: SQL Model <code><sub>ex</sub></code>
-layout: documentation
-id: sql_model
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This example demonstrates a relatively complete implementation of natural language interface to a
- SQL database. This is a non-trivial example, spanning over a thousand lines of code, and it can
- act as a basis for production-ready implementation.
- </p>
- <p>
- Note that a significant part of the implementation is not directly related to NLPCraft but rather deals
- with SQL statement construction. We specifically decided not to use any 3rd party libraries for it to
- show what can be done "from scratch".
- </p>
- <p>
- <b>Complexity:</b> <span class="complexity-three-star"><i class="fas fa-gem"></i> <i class="fas fa-gem"></i> <i class="fas fa-gem"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="background">
- <h2 class="section-title">Background <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many of modern existing natural language-to-SQL implementations use variations of
- deep learning approach where you first train the neural network on a pre-created training set and
- then get to use the trained network to provide (infer) probabilistic answers for the new input sentences. Although the latest
- natural language-to-SQL attempts to implement this approach are getting into 90% percentile of accuracy - they remain largely
- unusable for the vast swath of enterprise applications where non-deterministic nature of such systems
- renders them practically useless. The apparent problem with non-deterministic systems like that is that the user
- never knows whether a given answer is correct or incorrect. In these use cases users can't tolerate the fact that
- in 5-10% of the cases the answer will be invalid. Try that for your HR or revenue reporting system,
- prescriptive analytics systems, and so on...
- </p>
- <p>
- It's important to note, however, that for many non-mission-critical systems such non-determinism does
- not pose a significant problem. We happily accept such imprecision when asking for direction on our
- mobile devices, unlock our phones using face or fingerprint recognition, when performing sentiment
- analysis or trying to detect faces of our friends across thousands of photographs. Cost of retries,
- as well as the cost of initial errors, is insignificant in these cases. The same cost, however, in many business
- applications can be too significant to tolerate.
- </p>
- <p>
- As you may have learned by now, NLPCraft uses a <i>fully deterministic approach</i> in an attempt to
- match the user input against a defined set of intents. If it finds the matching intent - it guarantees that
- match for a given intent. If no matching intent can be found - it returns the negative
- result without any ambiguity. In other words, if the answer is given, it is deterministically guaranteed to be correct.
- Another positive side-effect of this approach is the fact that such matching logic is traceable, i.e.
- the user can see why given user input was matched against a certain intent (and not any other). Such
- traceability of the comprehension logic (or explainability vs "black box" approach from deep
- learning techniques) is often critical for many real-life business applications.
- </p>
- <p>
- On the flip side, one of the downsides of such an approach in general is the need to have a detailed,
- domain-specific model
- for each SQL database (when dealing with SQL databases). Building such a
- model can be a non-trivial and time consuming experience. That is where NLPCraft brings a lot of built-in tooling
- to simplify this task dramatically.
- </p>
- <div class="bq info">
- <b>Source Code</b>
- <p>
- Due to size of this example the entire source code for it can be found on <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">GitHub</a>.
- </p>
- </div>
- </section>
- <section id="sql">
- <h2 class="section-title">Sample Database <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to be building natural language interface against slightly modified "Northwind" sample database
- from <a target="_" href="https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/linq/downloading-sample-databases#get-the-northwind-sample-database-for-sql-server">Microsoft SQL Server</a>:
- </p>
- <figure>
- <img class="img-fluid" src="/images/sql_example_model.png" alt="">
- <figcaption><b>Fig 1.</b> SQL Schema</figcaption>
- </figure>
- <p>
- You can find SQL script creating this database and populating it with the sample data at
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/northwind.sql" target="github"><code>db/northwind.sql</code></a>
- file in the root of the example.
- </p>
- </section>
- <section id="h2">
- <h2 class="section-title">H2 Database <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In our example we are using H2 database instance. For convenience, example provides a command line application
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/db/SqlServer.scala" target="github"><code>db/SqlServer.scala</code></a>
- that automatically starts local H2 database instance with default configuration (localhost on port 9092) and initializes it
- using <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/northwind.sql" target="github"><code>db/northwind.sql</code></a> script.
- </p>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Creating data model manually for the existing SQL schema can be a daunting task. NLPCraft provides the tool
- that can scan SQL schema and create working stub of such model automatically. All you have to do then is to
- add any necessary modifications to it.
- </p>
- <p>
- <a href="/tools/sql_model_gen.html">SQL Model Generation</a> is a Java-based utility that takes
- JDBC configuration, reads database schema using it and creates initial JSON or YAML stub for the
- data model. This stub then can be used as is or be extended further.
- </p>
- <p>
- Examples specifically comes with two pre-generated files:
- </p>
- <ul>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/resources/sql_model_init.yaml" target="github"><code>sql_model_init.yaml</code></a> - the initial file that was generated straight out of
- SQL Model Generator (see above for the instructions).
- </li>
- </ul>
- </section>
- <section id="impl">
- <h2 class="section-title">Implementation <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Implementation mainly consists of these files:
- </p>
- <ul>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/SqlModel.scala" target="github"><code>SqlModel.scala</code></a> - the code behind the data model that loads YAML-defined model and
- defines all intents.
- </li>
- <li>
- <a href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft-examples/sql/src/main/java/org/apache/nlpcraft/examples/sql/db/SqlBuilder.scala" target="github"><code>db/SqlBuilder.scala</code></a> - the main utility that takes object model provided by
- <a href="/tools/sql_model_gen.html">SQL Model Generation</a>
- and builds a SQL query to execute. Note that we elected to build this functionality
- from scratch to illustrate how it can be done. You are free, of course, to use many
- of the existing libraries to help achieve this goal.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Assuming that project is located in <code>~/SqlModel</code> folder - let's go to into this directory and
- run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/SqlModel
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component
- that you generally need to start it only once for this and any other examples.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/sql/target/classes --mdls=demo.SqlModel
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created SQL model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#background">Background</a></li>
- <li><a href="#sql">Sample Database</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#impl">Implementation</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/examples/weather_bot.html b/examples/weather_bot.html
deleted file mode 100644
index f188701..0000000
--- a/examples/weather_bot.html
+++ /dev/null
@@ -1,582 +0,0 @@
----
-active_crumb: Weather Bot <code><sub>ex</sub></code>
-layout: documentation
-id: weather_bot
-fa_icon: fa-cube
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column example">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This example demonstrates relatively complete NLI-based weather service with JSON output and a non-trivial
- intent matching logic. It uses <a target="new" href="https://openweathermap.org/api/one-call-api">OpenWeather</a>
- REST service for the actual weather information.
- </p>
- <div class="bq info">
- <p>
- <b>NOTE:</b> you must provide OpenWeather API key in <code>OWM_API_KEY</code> system property when running
- the data probe. See <a target=_ href="https://openweathermap.org/api">https://openweathermap.org/api</a> for more information.
- </p>
- </div>
- <p>
- <b>Complexity:</b> <span class="complexity-two-star"><i class="fas fa-square"></i> <i class="fas fa-square"></i> <i class="far fa-square"></i></span><br/>
- <span class="ex-src">Source code: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/weather">GitHub <i class="fab fa-fw fa-github"></i></a><br/></span>
- <span class="ex-review-all">Review: <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">All Examples at GitHub <i class="fab fa-fw fa-github"></i></a></span>
- </p>
- </section>
- <section id="new_project">
- <h2 class="section-title">Create New Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can create new Java project in many ways - we'll use <a href="/tools/script.html">NLPCraft CLI</a>
- to accomplish this task:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-prj-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-prj-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-prj-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-project --baseName=Weather --outputDir=~ --pkgName=demo --mdlType=json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- New project created in <code>/home/Weather</code> directory.
- </li>
- <li>
- <code>gen-project</code> command defaults to Java and Maven as its built tool.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-project</code> to get a full help on <code>gen-project</code> command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-prj-out" role="tabpanel">
- <p></p>
- <img alt="" class="img-fluid" src="/images/weather_bot_fig1.png">
- </div>
- </div>
- </section>
- <section id="model">
- <h2 class="section-title">Data Model <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We are going to start with declaring the static part of our model using JSON which we will later load using
- <code>NCModelFileAdapter</code> in our Java-based model implementation. Open <code>src/main/resources/<b>weather.json</b></code>
- and replace its content with the following JSON:
- </p>
- <pre class="brush: js, highlight: [10, 18, 28, 38]">
-{
- "id": "nlpcraft.weather.ex",
- "name": "Weather Example Model",
- "version": "1.0",
- "description": "Weather example model.",
- "macros": [
- ],
- "elements": [
- {
- "id": "wt:phen",
- "description": "Weather phenomenon.",
- "synonyms": [
- "{high sea|severe weather|hail|heat wave|cold wave|derecho|supercell|avalanche|cyclone|wildfire|landslide|firestorm|dust storm|thunder snow|winter storm|cloudburst|shower|condensation|precipitation|drizzle|rainstorm|rain storm|rainfall|rain|storm|sun|sunshine|cloud|hot|cold|dry|wet|wind|hurricane|typhoon|sand-storm|sand storm|tornado|humid|fog|snow|smog|black ice|haze|thundershower|thundersnow|sleet|drought|wildfire|blizzard|avalanche|mist|thunderstorm}",
- "{weather {condition|temp|temperature|data|_}|condition|temp|temperature}"
- ]
- },
- {
- "id": "wt:hist",
- "description": "History (past) indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{history|past|previous}"
- ]
- },
- {
- "id": "wt:curr",
- "description": "Current indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{current|today|now|right now}"
- ]
- },
- {
- "id": "wt:fcast",
- "description": "Forecast (future) indicator.",
- "groups": [
- "indicator"
- ],
- "synonyms": [
- "{future|forecast|prognosis|prediction}"
- ]
- }
- ]
-}
- </pre>
- <p>There are number of important points here:</p>
- <ul>
- <li>
- <code>Line 10</code> defines an element <code>wt:phen</code> for various weather phenomenon.
- </li>
- <li>
- <code>Line 18</code> defines an element <code>wt:hist</code> whose presence will indicate the
- request for the past (history) weather information.
- </li>
- <li>
- <code>Line 28</code> defines an element <code>wt:curr</code> whose presence will indicate the
- request for the current (local) weather information.
- </li>
- <li>
- <code>Line 38</code> defines an element <code>wt:fcast</code> whose presence will indicate the
- request for the future (forecast) weather information.
- </li>
- </ul>
- <p>
- Now that our model is ready let's create a Java class that would load this model and define the intent
- that uses the model elements we have just defined.
- </p>
- <h2 class="section-title">Model Class <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Below is a full source code for our implementation. Note that this code uses several
- external tools for IP-based geo-location and the weather information
- provider (<a target=_ href="https://openweathermap.org/api">OpenWeather</a>). Note also that despite its apparent simplicity the model logic implementation is non-trivial.
- Significant portion of the code deals with a complex <b>temporal and geographical ambiguity</b>, i.e. the sentences like the these:
- </p>
- <dl>
- <dt>
- <code>'show my local weather'</code> vs <code>'show local Boston weather'</code>
- </dt>
- <dd>
- In the first sentence the word <code>local</code> indicates that the user is asking about her local weather
- (i.e. the weather for the location based on the IP address of the user's REST agent). However, in the
- second sentence the same word <code>local</code> is effectively ignored as user clearly indicated the city
- for the weather request (Boston).
- </dd>
- <dt>
- <code>'show weather forecast'</code> vs <code>'show weather forecast for today'</code> vs <code>'show last week weather forecast'</code>
- </dt>
- <dd>
- In these three sentences the word <code>forecast</code> defines future, current and past timeframe for the
- weather request. It acts as a future indicator in the first sentence and is effectively ignored in the second
- and third sentences.
- </dd>
- </dl>
- <div class="bq info">
- <p><b>Temporal <i class="amp">&</i> Geographical Ambiguity</b></p>
- <p>
- Despite seeming triviality of these ambiguities for the human comprehension, these represent a significant problem
- for the most deep learning neural networks unless specifically trained to resolve these particular cases. Procedural
- (deterministic) approach - as shown below - often yields a dramatically simpler and more robust solution, albeit the
- one that works only in a specific context.
- </p>
- </div>
- <p>
- Open <code>src/main/java/demo/<b>Weather.java</b></code> file and replace its content with the
- following code:
- </p>
- <pre class="brush: java, highlight: [30, 43, 130, 90, 132, 133, 134, 177]">
-package demo;
-
-import com.google.gson.Gson;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.nlpcraft.examples.weather.openweathermap.OpenWeatherMapException;
-import org.apache.nlpcraft.examples.weather.openweathermap.OpenWeatherMapService;
-import org.apache.nlpcraft.utils.keycdn.GeoManager;
-import org.apache.nlpcraft.utils.keycdn.beans.GeoDataBean;
-import org.apache.nlpcraft.model.NCIntent;
-import org.apache.nlpcraft.model.NCIntentMatch;
-import org.apache.nlpcraft.model.NCIntentSample;
-import org.apache.nlpcraft.model.NCIntentTerm;
-import org.apache.nlpcraft.model.NCModelFileAdapter;
-import org.apache.nlpcraft.model.NCRejection;
-import org.apache.nlpcraft.model.NCResult;
-import org.apache.nlpcraft.model.NCToken;
-
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static java.time.temporal.ChronoUnit.DAYS;
-
-public class WeatherModel extends NCModelFileAdapter {
- // System property for OpenWeatherMap API key.
- public final String OWM_API_KEY = "OWM_API_KEY";
-
- // Please register your own account at https://openweathermap.org/api and
- // replace this demo token with your own.
- // We are using the One Call API (https://openweathermap.org/api/one-call-api) in this example
- private final OpenWeatherMapService openWeather;
-
- private final GeoManager geoMrg = new GeoManager();
- private static final int DAYS_SHIFT_BACK = 5;
- private static final int DAYS_SHIFT_FORWARD = 7;
- private static final Gson GSON = new Gson();
- private static final Set<String> LOCAL_WORDS = new HashSet<>(Arrays.asList("my", "local", "hometown"));
-
- private Pair<Double, Double> prepGeo(NCIntentMatch ctx, Optional<NCToken> geoTokOpt) throws NCRejection {
- if (geoTokOpt.isPresent()) {
- NCToken geoTok = geoTokOpt.get();
-
- Map<String, Object> cityMeta = geoTok.meta("nlpcraft:city:citymeta");
-
- Double lat = (Double)cityMeta.get("latitude");
- Double lon = (Double)cityMeta.get("longitude");
-
- if (lat == null || lon == null) {
- String city = geoTok.meta("nlpcraft:city:city");
-
- throw new NCRejection(String.format("Latitude and longitude not found for: %s", city));
- }
-
- return Pair.of(lat, lon);
- }
-
- Optional<GeoDataBean> geoOpt = geoMrg.get(ctx.getContext().getRequest());
-
- if (geoOpt.isEmpty())
- throw new NCRejection("City cannot be determined.");
-
- // Manually process request for local weather. We need to separate between 'local Moscow weather'
- // and 'local weather' which are different. Basically, if there is word 'local/my/hometown' in the user
- // input and there is no city in the current sentence - this is a request for the weather at user's
- // current location, i.e. we should implicitly assume user's location and clear conversion context.
- // In all other cases - we take location from either current sentence or conversation STM.
-
- // NOTE: we don't do this separation on intent level as it is easier to do it here instead of
- // creating more intents with almost identical callbacks.
-
- @SuppressWarnings("SuspiciousMethodCalls")
- boolean hasLocalWord =
- ctx.getVariant().stream().anyMatch(t -> LOCAL_WORDS.contains(t.meta("nlpcraft:nlp:origtext")));
-
- if (hasLocalWord)
- // Because we implicitly assume user's current city at this point we need to clear
- // 'nlpcraft:city' tokens from conversation since they would no longer be valid.
- ctx.getContext().getConversation().clearStm(t -> t.getId().equals("nlpcraft:city"));
-
- // Try current user location.
- GeoDataBean geo = geoOpt.get();
-
- return Pair.of(geo.getLatitude(), geo.getLongitude());
- }
-
- @NCIntent(
- "intent=req " +
- "term~{tok_id() == 'wt:phen'}* " + // Zero or more weather phenomenon.
- "term(ind)~{" +
- "@isIndicator = has(tok_groups(), 'indicator') " + // Just to demo term variable usage.
- "@isIndicator" +
- "}* " + // Optional indicator words (zero or more).
- "term(city)~{tok_id() == 'nlpcraft:city'}? " + // Optional city.
- "term(date)~{tok_id() == 'nlpcraft:date'}?" // Optional date (overrides indicator words).
- )
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "Current forecast?",
- "Chance of rain in Berlin now?"
- })
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "Moscow forecast?",
- "Chicago history"
- })
- // NOTE: each samples group will reset conversation STM during auto-testing.
- @NCIntentSample({
- "What's the local weather forecast?",
- "What's the weather in Moscow?",
- "What's the current forecast for LA?",
- "What is the weather like outside?",
- "How's the weather?",
- "What's the weather forecast for the rest of the week?",
- "What's the weather forecast this week?",
- "What's the weather out there?",
- "Is it cold outside?",
- "Is it hot outside?",
- "Will it rain today?",
- "When it will rain in Delhi?",
- "Is there any possibility of rain in Delhi?",
- "Is it raining now?",
- "Is there any chance of rain today?",
- "Was it raining in Beirut three days ago?",
- "How about yesterday?"
- })
- public NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("ind") List<NCToken> indToksOpt,
- @NCIntentTerm("city") Optional<NCToken> cityTokOpt,
- @NCIntentTerm("date") Optional<NCToken> dateTokOpt
- ) {
- try {
- Instant now = Instant.now();
-
- Instant from = now;
- Instant to = now;
-
- if (indToksOpt.stream().anyMatch(tok -> tok.getId().equals("wt:hist")))
- from = from.minus(DAYS_SHIFT_BACK, DAYS);
- else if (indToksOpt.stream().anyMatch(tok -> tok.getId().equals("wt:fcast")))
- to = from.plus(DAYS_SHIFT_FORWARD, DAYS);
-
- if (dateTokOpt.isPresent()) { // Date token overrides any indicators.
- NCToken dateTok = dateTokOpt.get();
-
- from = Instant.ofEpochMilli(dateTok.meta("nlpcraft:date:from"));
- to = Instant.ofEpochMilli(dateTok.meta("nlpcraft:date:to"));
- }
-
- Pair<Double, Double> latLon = prepGeo(ctx, cityTokOpt); // Handles optional city too.
-
- double lat = latLon.getLeft();
- double lon = latLon.getRight();
-
- return NCResult.json(GSON.toJson(from == to ? openWeather.getCurrent(lat, lon) : openWeather.getTimeMachine(lat, lon, from, to)));
- }
- catch (OpenWeatherMapException e) {
- throw new NCRejection(e.getLocalizedMessage());
- }
- catch (NCRejection e) {
- throw e;
- }
- catch (Exception e) {
- throw new NCRejection("Weather provider error.", e);
- }
- }
-
- /**
- * Loads the model.
- */
- public WeatherModel() {
- // Load model from external JSON file on classpath.
- super("weather_model.json");
-
- // Try system variable first.
- String apiKey = System.getProperty(OWM_API_KEY);
-
- if (apiKey == null)
- // Try environment variable next.
- apiKey = System.getenv(OWM_API_KEY);
-
- if (apiKey == null)
- throw new OpenWeatherMapException(String.format("Provide OpenWeatherMap API key using '-D%s=<your-key-here>' system property.", OWM_API_KEY));
-
- openWeather = new OpenWeatherMapService(apiKey, 5, 7);
- }
-
- @Override
- public void onDiscard() {
- openWeather.stop();
- }
-}
- </pre>
- <ul>
- <li>
- <code>Line 177</code> loads the model configuration from the external <code>weather_model.json</code>
- file.
- </li>
- <li>
- Method <code>preGeo(...)</code> on the <code>line 43</code> handles the geolocation processing including
- IP-based geo-location and resolution of the geographical ambiguity.
- </li>
- <li>
- <code>Line 130</code> defines a callback for the intent defined on the <code>line 90</code>. Note that
- callback implementation also deals with the temporal ambiguity.
- </li>
- <li>
- <code>Lines 132-134</code> define formal callback method parameters that correspond
- to the intent's terms (see <code>line 90</code>).
- </li>
- <li>
- <code>Lines 101-129</code> define input sentence samples that are used for both documentation and
- auto-validation purposed (see <a href="#testing">testing</a> second for details).
- </li>
- <li>
- Make sure to provide you <a target="new" href="https://openweathermap.org/api/one-call-api">OpenWeather API</a>
- key on the <code>line 30</code>.
- </li>
- </ul>
- </section>
- <section id="build_project">
- <h2 class="section-title">Build Project <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Once we have our model ready let's go to <code>~/Weather</code> directory and run the Maven build:
- </p>
- <pre class="brush: bash">
- $ cd ~/Weather
- $ mvn clean package
- </pre>
- <p>
- At this stage we have our project built and we are ready to start testing.
- </p>
- </section>
- <section id="start_server">
- <h2 class="section-title">Start Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Run the following command to start local REST server, if it hasn't been started already, from the NLPCraft installation directory:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-cmd" role="tab">Command</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-out" role="tab">Output <i class="fa fa-desktop output"></i></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-cmd" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-srv-out" role="tabpanel">
- <p></p>
- <p>
- <img class="img-fluid" alt="" src="/images/server-fig1.png">
- </p>
- </div>
- </div>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <i style="color: #F39C12" class="fa fa-exclamation-triangle"></i> REST server is a "fire-and-forget" component that you generally need to start it only once.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- </section>
- <section id="testing">
- <h2 class="section-title">Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Remember the <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- annotation we have used in our model code next to intent definition?
- </p>
- <p>
- Part of the <a href="/tools/test_framework.html">test framework</a>, the auto-validator class <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> takes one or more model IDs
- (or class names) and performs validation. Validation consists of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model,
- scanning for <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations
- and their corresponding callback methods, submitting each sample input
- sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- Note that auto-testing does not require any additional code to be written - the class gathers all required information from the model
- itself.
- </p>
- <p>
- As always, you can launch model auto-validator as any other Java class but we'll use NLPCraft CLI
- to do it more conveniently:
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh test-model --cp=~/Weather/target/classes --mdls=demo.Weather
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- Note that you can use <code>retest-model</code> command in REPL mode to re-run the last model test
- avoiding the retyping of all required parameters.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- </ul>
- <p>
- Look at the output of this command and you will see the test results for all our sample utterances:
- </p>
- <p>
- <img style="max-width: 871px !important;" class="img-fluid" alt="" src="/images/weather-bot-test.png">
- </p>
- </section>
- <section id="rinse">
- <h2 class="section-title">Rinse <span class="amp">&</span> Repeat <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Typical development cycle consists of:
- </p>
- <ul>
- <li>
- <a href="#model">Modifying the model</a>
- </li>
- <li>
- <a href="#build_project">Re-building the project</a>
- </li>
- <li>
- <a href="#testing">Re-running the test</a>
- </li>
- </ul>
- <p>
- All of these operations can be performed from <a href="/tools/script.html">NLPCraft CLI</a> in REPL mode or from any IDE.
- </p>
- <p>
- NOTE: you don't need to restart REST server every time - it only needs to be started once.
- </p>
- </section>
- <section>
- <h2 class="section-title">Done! 👌 <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You've created weather bot data model, started the REST server and tested this model using the built-in test framework.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#new_project">New Project</a></li>
- <li><a href="#model">Data Model</a></li>
- <li><a href="#build_project">Build Project</a></li>
- <li><a href="#start_server">Start Server</a></li>
- <li><a href="#testing">Testing</a></li>
- <li><a href="#rinse">Rinse <span class="amp">&</span> Repeat</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/getting-started.html b/getting-started.html
index c4c6d8c..e88aa3b 100644
--- a/getting-started.html
+++ b/getting-started.html
@@ -48,12 +48,6 @@
</ul>
</section>
<section>
- <h2 class="section-title">Weather Forecast <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- We'll be testing <a href="/examples/weather_bot.html">Weather Example</a>
- to ask questions about weather forecast using REST APIs. This example returns a JSON weather
- data for variety of different inquiries about the past, present or future weather conditions.
- </p>
<h2 id="probe-server" class="section-sub-title">Data Probe <span class="amp">&</span> REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
<p>
<a href="/server-and-probe.html">Data probes</a> are used to deploy and host data model, while <a href="/server-and-probe.html">REST server</a> (or a
diff --git a/metrics-and-tracing.html b/metrics-and-tracing.html
deleted file mode 100644
index d8cc82a..0000000
--- a/metrics-and-tracing.html
+++ /dev/null
@@ -1,307 +0,0 @@
----
-active_crumb: Metrics <span class="amp">&</span> Tracing
-layout: documentation
-id: metrics
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div id="metrics-and-tracing" class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both REST server and data probe come with <a target=_ href="https://opencensus.io">OpenCensus</a>
- instrumentation. NLPCraft bundles several built-in exporters for various OpenCensus trace and stats
- backends:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Backend</th>
- <th>Built-In Exporters</th>
- <th>Stats or Trace?</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td>
- <a target=_ href="https://www.jaegertracing.io/">
- <img alt="" style="width: 48px" src="/images/jaeger-logo-x.png">
- <div class="exporter-name">Jaeger</div>
- </a>
- </td>
- <td>
- Server: <code>NCJaegerExporter</code><br>
- Probe:
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCJaegerExporter.html">NCJaegerExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://zipkin.io/">
- <img alt="" style="width: 48px" src="/images/zipkin-logo.jpg">
- <div class="exporter-name">Zipkin</div>
- </a>
- </td>
- <td>
- Server: <code>NCZipkinExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCZipkinExporter.html">NCZipkinExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://prometheus.io/">
- <img alt="" style="width: 48px" src="/images/prometheus-logo-x.png">
- <div class="exporter-name">Prometheus</div>
- </a>
- </td>
- <td>
- Server: <code>NCPrometheusExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://grafana.com/">
- <img alt="" style="width: 48px" src="/images/grafana-logo-x.png">
- <div class="exporter-name">Grafana<sup>*</sup></div>
- </a>
- </td>
- <td>
- Server: <code>NCPrometheusExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://cloud.google.com/stackdriver/">
- <img alt="" style="width: 48px" src="/images/stackdriver-stats-x.png">
- <div class="exporter-name">Stackdriver</div>
- </a>
- </td>
- <td>
- Server: <code>NCStackdriverStatsExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverStatsExporter.html">NCStackdriverStatsExporter</a>
- </td>
- <td style="text-align: center;">Stats</td>
- </tr>
- <tr>
- <td>
- <a target=_ href="https://cloud.google.com/stackdriver/">
- <img alt="" style="width: 48px" src="/images/stackdriver-trace-x.png">
- <div class="exporter-name">Stackdriver</div>
- </a>
- </td>
- <td>
- Server: <code>NCStackdriverTraceExporter</code><br>
- Probe: <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverTraceExporter.html">NCStackdriverTraceExporter</a>
- </td>
- <td style="text-align: center;">Trace</td>
- </tr>
- </tbody>
- </table>
- <p>
- <sup><b>NOTE:</b> (*) Grafana works through Prometheus integration.</sup>
- </p>
- <p>
- All built-in exporters are a simple adaptation of the standard OpenCensus <a target=_ href="https://opencensus.io/exporters/supported-exporters/java/">exporters</a>
- to life cycle components for the
- REST server or the data probe. Each exporter has a few lines of code and you can create your own
- exporters for other backends.
- </p>
- </section>
- <section id="server">
- <h2 class="section-title">Configuring Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- OpenCensus exporters for the REST server are lifecycle components. Server lifecycle components are the implementations
- of <code>NCServerLifecycle</code> trait. Lifecycle components get callbacks for various stages of the server
- lifecycle and thus can register and unregister "primordial" components such as OpenCensus exporters.
- </p>
- <p>
- REST server lifecycle components are specified in <code>nlpcraft.server.lifecycle</code>
- <a href="server-and-probe.html">configuration</a> property. For example, for file based configuration:
- </p>
- <pre class="brush: js, highlight: [17, 18]">
-nlpcraft {
- server {
- ...
- lifecycle = [
- "org.apache.nlpcraft.server.lifecycle.opencensus.NCPrometheusExporter",
- "org.apache.nlpcraft.server.lifecycle.opencensus.NCJaegerExporter"
- ]
- ...
- }
-}
- </pre>
- <p>
- Note that each server exporter has default configuration that can be overridden via
- <a href="server-and-probe.html">configuration</a> properties or environment variables. Note also that all
- built-in <b>trace exporters</b> use "always" sampling by default that is suitable
- only for demo or development purposes. For production usage you will need to modify the exporter
- to provide more effective <a target=_ href="https://opencensus.io/tracing/sampling/">sampling strategy</a>.
- </p>
- </section>
- <section id="probe">
- <h2 class="section-title">Configuring Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Just like on the REST server OpenCensus exporters for the data probe are lifecycle components. Probe lifecycle
- components are the implementations
- of <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCLifecycle.html">org.apache.nlpcraft.model.NCLifecycle</a>
- interface. Lifecycle components get callbacks for various stages of the data probe
- lifecycle and thus can register and unregister "primordial" components such as OpenCensus exporters. For example, using
- file based configuration:
- </p>
- <pre class="brush: js, highlight: [14, 15]">
-nlpcraft {
- probe {
- ...
- lifecycle = [
- "org.apache.nlpcraft.model.opencensus.NCPrometheusExporter",
- "org.apache.nlpcraft.model.opencensus.NCJaegerExporter"
- ]
- ...
- }
-}
- </pre>
- <p>
- Just like with server exporters, each probe exporter has default configuration that can be overridden via
- <a href="server-and-probe.html">configuration</a> properties or environment variables. Note also that all
- built-in <b>trace exporters</b> use "always" sampling by default that is suitable
- only for demo or development purposes. For production usage you will need to modify the exporter
- to provide more effective <a target=_ href="https://opencensus.io/tracing/sampling/">sampling strategy</a>.
- Please consult individual exporter classes:
- </p>
- <ul>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCJaegerExporter.html">NCJaegerExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCZipkinExporter.html">NCZipkinExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCPrometheusExporter.html">NCPrometheusExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverStatsExporter.html">NCStackdriverStatsExporter</a></li>
- <li><a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/opencensus/NCStackdriverTraceExporter.html">NCStackdriverTraceExporter</a></li>
- </ul>
- </section>
- <section id="traces">
- <h2 class="section-title">Traces <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When traces are exporter, there are two service names used for NLPCraft traces that will appear on UI backends:
- </p>
- <ul>
- <li><code>nlpcraft-server</code> for traces from REST server.</li>
- <li><code>nlpcraft-probe</code> for traces from data probes.</li>
- </ul>
- <p>
- All traces are extensively tagged with various information. Specifically, watch out for <code>srvReqId</code>,
- <code>modelId</code> and <code>userId</code> tags that will help you navigate across traces pertaining to specific
- request, models or users.
- </p>
- </section>
- <section id="metrics">
- <h2 class="section-title">Metrics <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When it comes to metrics each exporter has different set of configuration for either a namespace or
- name prefix on how these metrics will be available on the UI backends:
- </p>
- <ul>
- <li>
- <a target=_ href="https://prometheus.io/">Prometheus</a> and <a target=_ href="https://grafana.com">Grafana</a>
- stats exporters by default use <code>nlpcraft-server</code> and <code>nlpcraft-probe</code> namespaces.
- </li>
- <li>
- <a target=_ href="https://cloud.google.com/stackdriver/">Stackdriver</a> stats exporters use default
- <code>custom.googleapis.com/nlpcraft/server</code> and <code>custom.googleapis.com/nlpcraft/probe</code> metric
- name prefixes.
- </li>
- </ul>
- <p>
- <b>NOTE:</b> default namespaces and service prefixes can be changed via <a href="server-and-probe.html">configuration</a>
- properties or environment variables
- </p>
- <p>
- The following metrics are generated by the REST server:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Name <sub>without prefix or namespace</sub></th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- {% for api in site.data.metrics.server %}
- <tr>
- <td>
- <code>{{api.name}}_latdist_bucket</code><br>
- <code>{{api.name}}_latdist_count</code><br>
- <code>{{api.name}}_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the <b>{{api.description}}</b> latencies.</td>
- </tr>
- {% endfor %}
- <tr>
- <td>
- <code>roundtrip_latdist_bucket</code><br>
- <code>roundtrip_latdist_count</code><br>
- <code>roundtrip_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the full server-probe-server round trip latencies.</td>
- </tr>
- </tbody>
- </table>
- <p>
- The following metrics are generated by the data probe:
- </p>
- <table class="gradient-table checks">
- <thead>
- <tr>
- <th>Name <sub>without prefix or namespace</sub></th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- {% for api in site.data.metrics.probe %}
- <tr>
- <td>
- <code>{{api.name}}_latdist_bucket</code><br>
- <code>{{api.name}}_latdist_count</code><br>
- <code>{{api.name}}_latdist_sum</code>
- </td>
- <td>Histogram, count and sum of values for the <b>{{api.description}}</b> latencies.</td>
- </tr>
- {% endfor %}
- </tbody>
- </table>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#server">Configuring Server</a></li>
- <li><a href="#probe">Configuring Probe</a></li>
- <li><a href="#traces">Traces</a></li>
- <li><a href="#metrics">Metrics</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
diff --git a/server-and-probe.html b/server-and-probe.html
deleted file mode 100644
index d2e611c..0000000
--- a/server-and-probe.html
+++ /dev/null
@@ -1,618 +0,0 @@
----
-active_crumb: Server <span class="amp">&</span> Probe
-layout: documentation
-id: server_and_probe
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div id="server-and-probes" class="col-md-8 second-column">
- <section>
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server and data probe are the two runtime components that you need to run when using NLPCraft.
- Data probes are used to deploy and host data model, while REST server (or a cluster of servers) is used
- to accept client REST call and route them to the data models via data probes.
- </p>
- <h2 class="section-sub-title">REST Server vs. Data Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- It's important to remember why REST server is a separate component from a data probe. While a
- typical deployment would have only one REST server (or a cluster of REST servers behind a single
- load balancer), there are maybe multiple data probes hosting different data models deployed in
- different physical locations, managed through different life cycles and requiring different
- security and network configurations.
- </p>
- <p>
- Moreover, REST server is a heavy and resource consuming component that is built around Apache
- Ignite distributing in-memory computing capabilities - while the data probe is a lightweight
- data model container. During the development and testing of data models, the developers need to
- frequently redeploy data models by restarting the data probe. If the REST server and the data probe
- would be one component - this process would be very inefficient.
- </p>
- <div class="bq success">
- <div class="bq-idea-container">
- <div><div>💡</div></div>
- <div>
- <p><b>Configuration</b></p>
- <p>
- Both REST server and the data probe can share their configuration file or be configured
- individually. Read more about this in <a href="#config">configuration</a>
- section.
- </p>
- </div>
- </div>
- </div>
- <h2 class="section-sub-title">All-Inclusive JAR <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- <a href="/download.html#zip">Binary</a> NLPCraft ZIP download comes with a single executable JAR file that includes all
- necessary dependencies (except for examples): <code>build/<b>apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar</b></code>.
- This single all-inclusive JAR file can be used to start any NLPCraft runtime components as standard
- Java applications:
- </p>
- <ul>
- <li><a href="#server">REST Server</a></li>
- <li><a href="#probe">Data Probe</a></li>
- <li><a href="/tools/script.html">NLPCraft CLI</a></li>
- <li><a href="/data-model.html">Model API</a></li>
- </ul>
- <p>
- Note that if you downloaded the <a href="/download.html#src">source</a> ZIP you need to run <code class="script">mvn clean package -P examples</code> to
- get the <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code>
- file. It will be located in <code>nlpcraft/target</code> sub-folder.
- </p>
- <h2 class="section-sub-title">Examples JARs <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- If you downloaded binary release the examples JARs are pre-built and shipped within it. They are located in <code>build/nlpcraft-examples/xxx</code> folder
- for each <code>xxx</code> example. If you downloaded a source release you will need to run <code class="script">mvn clean package -P examples</code>
- and examples JARs will be located in each individual module under its <code>target</code> sub-folder.
- </p>
- </section>
- <section id="server">
- <h2 class="section-title">REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server accepts client REST calls and routes them to the data model hosted by data probes.
- REST server can be started in different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-srv-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-srv-docker" role="tab"><i class="fab fa-docker"></i> Docker</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-srv-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-server
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-server</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-srv-class" role="tabpanel">
- <p></p>
- <p>
- If using executable JAR:
- </p>
- <pre class="brush: bash">
- $ java -Xms1024m -jar apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar -server
- </pre>
- <p>
- If specifying additional classpath components and need <code>-cp</code> parameter:
- </p>
- <pre class="brush: bash">
- $ java -Xms1024m -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.NCStart -server
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Make sure to provide correct path to <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> is a common entry point for all NLPCraft runtime components and
- can be used to start REST server from IDE.
- </li>
- </ul>
- <p>
- <b>Parameters</b>:
- </p>
- <dl>
- <dt>
- <code>-server</code>
- </dt>
- <dd>
- <em>Mandatory</em> parameter to indicate that you are starting the REST server.
- </dd>
- <dt><code>-config=path</code></dt>
- <dd>
- <em>Optional</em> parameter to provide configuration file path.
- Server will automatically look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file. If the configuration
- file has different name or in different location use <code>-config=path</code> parameter
- where <code>path</code> is an absolute path to the configuration file. Note that the server and the data
- probe can use the same file for their configuration (like the
- default <code>nlpcraft.conf</code> contains configuration for both the server and the data probe).
- </dd>
- <dt><code>-igniteConfig=path</code></dt>
- <dd>
- <em>Optional</em> parameter to provide <a target=_ href="https://ignite.apache.org/">Apache Ignite</a> configuration file path.
- Note that Apache Ignite is used as a cluster computing plane and a default distributed storage.
- Server will automatically look for <code>ignite.xml</code>
- configuration file in the same directory as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- If the configuration file has different name or in different location use <code>-igniteConfig=path</code> parameter
- where <code>path</code> is an absolute path to the Ignite configuration file.
- </dd>
- </dl>
- <p>
- <b>VM Options</b>:
- </p>
- <p>
- NLPCraft REST server uses Apache Ignite 2.x as its distributed in-memory computing plane. Apache Ignite
- <a target=_new href="https://apacheignite.readme.io/docs/getting-started#running-ignite-with-java-11-and-later-versions">requires</a>
- the following additional JVM options to be used when running Apache Ignite 2.x on JDK 11:
- </p>
- <pre class="brush: text">
- --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
- --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
- --add-opens=java.base/java.nio=ALL-UNNAMED
- --add-opens=java.base/java.io=ALL-UNNAMED
- --add-opens=java.base/java.util=ALL-UNNAMED
- --add-opens=java.base/java.lang=ALL-UNNAMED
- --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED
- --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED
- --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED
- --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
- --illegal-access=permit
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script automatically uses
- these options for <code>start-server</code> command.
- </li>
- </ul>
- <div class="bq success">
- Since this list of required VM options is long it is rather impractical and inconvenient to use it from the
- command line. It is <b>highly recommended</b> to either use IDE or <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script
- to manage REST server lifecycle.
- </div>
- </div>
- <div class="tab-pane fade show" id="nav-srv-docker" role="tabpanel">
- <p></p>
- <p>
- If Docker image is available for given version you can start REST server as follows:
- </p>
- <pre class="brush: bash">
- $ docker run -m 8G -p 8081:8081 -p 8201:8201 -p 8202:8202 nlpcraftserver/server:{{site.latest_version}}
- </pre>
- </div>
- </div>
- <h2 class="section-sub-title">JVM Memory <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Make sure to allocate enough memory for server JVM using <code>-Xms</code> JVM option, i.e. <code>-Xms1024m</code>.
- Many 3rd party NLP engines like Stanford CoreNLP are very memory intensive and may require several GBs
- of JVM heap allocated depending on the models used. Note that when server JVM has insufficient heap
- memory the Apache Ignite may throw the following warning logs:
- </p>
- <pre class="brush: text">
- Jul-22 13:27:56 [INFO ] ...
- Jul-22 13:28:08 [WARN ] Possible too long JVM pause: 11364 milliseconds.
- Jul-22 13:28:11 [INFO ] ...
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html"><code>nlpcraft.{sh|cmd}</code></a> script automatically uses
- <code>-Xms1024m</code> for <code>start-server</code> command.
- </li>
- </ul>
- <p>
- The abnormally long GC pauses (over 5s) can be caused by the excessive memory swapping performed by OS due to
- insufficient JVM heap memory.
- </p>
- </section>
- <section id="probe">
- <h2 class="section-title">Data Probe <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Data probes are used to deploy and host data mode, and can also be started in several ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-probe-script" role="tab" >NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-probe-class" role="tab" >Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-probe-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh start-probe --cp=/path/to/model/classes # Use default configuration.
- $ bin/nlpcraft.sh start-probe --cp=/path/to/model/classes --mdls=com.package.MyModel
- > restart-probe # Restart the probe with the last set of parameters in REPL mode.
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- <code>--cp</code> parameter must provide additional JVM classpath for models to deploy in this probe.
- </li>
- <li>
- Optional <code>--mdls</code> parameter can be used to specify a one or more specific models to deploy if more than
- one model is available.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=start-probe</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-probe-class" role="tabpanel">
- <p></p>
- <p>
- If using executable JAR:
- </p>
- <pre class="brush: bash">
- $ java -jar apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar -probe
- </pre>
- <p>
- If specifying additional classpath components and need <code>-cp</code> parameter:
- </p>
- <pre class="brush: bash">
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar:/my/project/classes org.apache.nlpcraft.NCStart -probe -config=/my/project/probe.conf
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <code>/my/project</code> directory contains user-defined model implementation
- </li>
- <li>
- Make sure to provide correct path to <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> is a common entry point for all NLPCraft runtime components.
- </li>
- <li>
- Class <code>org.apache.nlpcraft.NCStart</code> should be used to star data probe from IDE.
- </li>
- </ul>
- <p>
- <b>Parameters:</b>
- </p>
- <dl>
- <dt>
- <code>-probe</code>
- </dt>
- <dd>
- <em>Mandatory</em> parameter to indicate that you are starting a data probe.
- </dd>
- <dt><code>-config=path</code></dt>
- <dd>
- <p>
- <em>Optional</em> parameter to provide probe configuration file path.
- Data probe will automatically look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file. If the configuration
- file has different name or in different location use <code>-config=path</code> parameter
- where <code>path</code> is an absolute path to the data probe configuration file. Note that the server and the data
- probe can use the same file for their configuration (like the
- default <code>nlpcraft.conf</code> contains configuration for both the server and the data probe).
- </p>
- </dd>
- </dl>
- </div>
- </div>
- </section>
- <section id="config">
- <h2 class="section-title">Configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both REST server and the data probe use <a target=_ href="https://github.com/lightbend/config/">Typesafe Config</a> for their configuration:
- </p>
- <ul>
- <li>Both the server and the data probe come with default configuration available in <code>build/<b>nlpcraft.conf</b></code> file.</li>
- <li>
- By default, REST server is looking for <code>server.conf</code> and then <code>nlpcraft.conf</code> configuration file.
- </li>
- <li>
- By default, data probe is looking for <code>probe.conf</code> and then <code>nlpcraft.conf</code> configuration file.
- </li>
- <li>Custom configuration or default overrides can be placed into a file or provided via environment variables.</li>
- <li>Configuration files use <a target=_ href="https://github.com/lightbend/config/blob/master/HOCON.md">HOCON</a> file format.</li>
- <li>
- When server and probe use different configuration files - each file would have either <code>nlpcraft.server</code>
- or <code>nlpcraft.probe</code> sub-section.
- </li>
- </ul>
- <p>
- By default, when REST server or data probe start they look for <code>nlpcraft.conf</code> configuration file in the same directory
- as <code>apache-nlpcraft-incubating-<b>{{site.latest_version}}</b>-all-deps.jar</code> file and the on their classpath. You can change this behavior with
- <code>-config=path</code> parameter.
- </p>
- <div class="bq success">
- <div class="bq-idea-container">
- <div><div>💡</div></div>
- <div>
- <p><b>Configuration Example</b></p>
- <p>
- Default configuration is available in <code>build/<b>nlpcraft.conf</b></code> file and it is extensively
- documented including all optional parameters and default values.
- </p>
- </div>
- </div>
- </div>
- <h2 class="section-sub-title">Server <span class="amp">&</span> Probe Configurations <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Default configuration is available in <code>build/<b>nlpcraft.conf</b></code> file and it is extensively documented. It has subsections
- for the server and probe configuration. You can also separate server and probe configurations into their own
- separate files. While server and probe can use any file name for their configuration files, the server
- looks for <code>server.conf</code> file by default, and the probe looks for <code>probe.conf</code>
- file before searching for <code>nlpcraft.conf</code> file.
- Each such file would have a subsection of configuration for either server or data probe.
- </p>
- <p>
- Server configuration file (e.g. <code>server.conf</code>):
- </p>
- <pre class="brush: js">
-nlpcraft {
- server {
- ...
- }
-}
- </pre>
- <p>
- Probe configuration file (e.g. <code>probe.conf</code>):
- </p>
- <pre class="brush: js">
-nlpcraft {
- probe {
- ...
- }
-}
- </pre>
- </section>
- <section id="override">
- <h2 class="section-sub-title">Custom Configuration <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- While you can change configuration file or files for your own needs (and use <code>-config=...</code>
- parameter described above to provide path to that file) it is often more convenient to use the default configuration file and
- change one or two properties in it. You can accomplish this by using standard
- <a target=_ href="https://github.com/lightbend/config#optional-system-or-env-variable-overrides">HOCON overriding</a>
- via environment variables:
- </p>
- <ol>
- <li>
- Set probe or server JVM system property <code>-Dconfig.override_with_env_vars=true</code> which will instruct
- configuration framework to look for external overrides.
- </li>
- <li>For each configuration property <code>x.y.z</code> set the overriding environment variable <code>CONFIG_FORCE_x_y_z=some_value</code></li>
- <li>See more details on <a target=_ href="https://github.com/lightbend/config#optional-system-or-env-variable-overrides">HOCON documentation</a>.</li>
- </ol>
- <p>
- Consider the following snippet of NLPCraft configuration:
- </p>
- <pre class="brush: js">
-nlpcraft {
- probe {
- models = "com.nlp.MyModel"
- }
-
- server {
- lifecycle = "org.apache.nlpcraft.server.lifecycle.opencensus.NCJaegerExporter"
-
- rest {
- host = "0.0.0.0"
- port = 8081
- apiImpl = "org.apache.nlpcraft.server.rest.NCBasicRestApi"
- }
- }
-}
- </pre>
- <p>
- You can override these properties with the following environment variables:
- </p>
- <p>
- <code>CONFIG_FORCE_<b>nlpcraft_server_rest_host</b>=1.2.3.4</code><br>
- <code>CONFIG_FORCE_<b>nlpcraft_server_lifecycle</b>="org.nlp.Lifecycle1, org.nlp.Lifecycle1"</code><br>
- <code>CONFIG_FORCE_<b>nlpcraft_probe_models</b>="com.nlp.MyModel, com.nlp.AnotherModel"</code>
- </p>
- <div class="bq info">
- <b>Examples</b>
- <p>
- Note that all examples that come with NLPCraft have instructions that use environment variable overriding
- for running their data probes. They use default <code>nlpcraft.conf</code> file and override
- one <code>nlpcraft.probe.models</code> property (see above) to specify what model the data probe
- needs to deploy.
- </p>
- </div>
- </section>
- <section id="ansi">
- <h2 class="section-sub-title">ANSI Colors <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Both NLPCraft server and probe use ANSI coloring via
- <a target=_ href="https://en.wikipedia.org/wiki/ANSI_escape_code">ANSI escape sequences</a> for their log
- output by default. ANSI coloring provides easer console log comprehension and modern esthetics:
- </p>
- <p>
- <img class="non-fluid-img" src="/images/ansi_colors.png" alt="">
- </p>
- <p>
- However, there are
- cases when either specific console does not support ANSI escape sequences, or specific color schema
- isn't suitable or log being redirected to a file or piped to downstream system. In these cases you need to
- disable ANSI coloring to avoid polluting log with unprocessed ANSI escape codes.
- </p>
- <p>
- You can disable ANSI coloring in either server, probe or both by supplying the following system
- property to JVM process: <code>-D<b>NLPCRAFT_ANSI_COLOR_DISABLED</b>=true</code>
- </p>
- </section>
- <section id="testing">
- <h2 class="section-title">CI Testing <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- It is a good practice to run units tests during routine builds using Maven (or others CI toolchains). To
- test data models you need to have a running server and then start one or more data probes with
- models you want to test. While doing this from IDE can be trivial enough, doing this from Maven
- can be tricky.
- </p>
- <p>
- The challenge is that from the Maven build you need to start the server, wait til its fully started and
- initialized, and only then start issuing REST calls, start data probes or run tests that use <a href="/tools/embedded_probe.html">embedded probes</a>.
- When done manually (e.g. from IDE) you can visually observe when the server finished its startup and then manually
- launch the tests. In Maven, however, you need to use a special plugin to accomplish the same in
- automated fashion.
- </p>
- <div class="bq info">
- <b>Probe Waits For Server</b>
- <p>
- Technically, when a data probe starts up it will initialize, load the models, and will automatically wait for the server to get online
- if it isn't yet (as well as periodically check for it). Once server is online the data probe will automatically connect to it. However,
- if the unit tests don't use data probe and only issue REST calls then these tests have to somehow wait for the
- server to get online.
- </p>
- <p>
- To overcome this challenge you can use
- <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>
- Maven plugin.
- </p>
- </div>
- <p>
- To get around this problem NLPCraft uses <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>
- Maven plugin in its own build. This plugin allows to start the external process and use configured URL
- endpoint to check whether or not the external process has fully started. This works perfect with NLPCraft server
- <a href="/using-rest.html">health check REST call</a>. The plugin can be configured in the following way for your own project
- (taken directly from NLPCraft <a target="github" href="https://github.com/apache/incubator-nlpcraft/blob/master/nlpcraft/pom.xml"><code>pom.xml</code></a>):
- </p>
- <pre class="brush: xml, highlight: [14, 16, 32]">
-<plugin>
- <groupId>com.bazaarvoice.maven.plugins</groupId>
- <artifactId>process-exec-maven-plugin</artifactId>
- <version>0.9</version>
- <executions>
- <execution>
- <id>pre-integration-test</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>start</goal>
- </goals>
- <configuration>
- <name>server</name>
- <healthcheckUrl>http://localhost:8081/api/v1/health</healthcheckUrl>
- <waitAfterLaunch>180</waitAfterLaunch>
- <processLogFile>${project.build.directory}/server.log</processLogFile>
- <arguments>
- <argument>java</argument>
- <argument>-Xmx4G</argument>
- <argument>-Xms4G</argument>
- <argument>--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.nio=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.io=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.util=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/java.lang=ALL-UNNAMED</argument>
- <argument>--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED</argument>
- <argument>--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED</argument>
- <argument>--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED</argument>
- <argument>--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</argument>
- <argument>--illegal-access=permit</argument>
- <argument>-DNLPCRAFT_ANSI_COLOR_DISABLED=true</argument>
- <argument>-Djdk.tls.client.protocols=TLSv1.2</argument>
- <argument>-jar</argument>
- <argument>${project.build.directory}/${nlpcraft.all.deps.jar}</argument>
- <argument>-server</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>stop-all</id>
- <phase>post-integration-test</phase>
- <goals>
- <goal>stop-all</goal>
- </goals>
- </execution>
- </executions>
-</plugin>
- </pre>
- <p>
- <b>NOTES</b>:
- </p>
- <ul>
- <li>
- On line 14 we specify the URL endpoint to check whether or not our server is online. We use
- <code>/health</code> localhost REST call for that.
- </li>
- <li>
- On line 16 we redirect the output from server to a dedicated file to <b>avoid interleaving</b> log
- from server and log from data probe in the same console (where we are running the Maven build from).
- Such interleaving will make the combined log unreadable and can cause output problem for the console
- due to mixed up ANSI escape sequences from server and data probe.
- </li>
- <li>
- Since the server log is piped out to a separate file we disable ANSI coloring for the server
- on the line 32.
- </li>
- </ul>
- <div class="bq info">
- <b>Avoid Interleaving Logs</b>
- <p>
- When running both server and the data probe(s) from the Maven build it is important to avoid interleaving
- logs from the server and the probe. Such interleaving will make the combined log in Maven unreadable and
- can cause console malfunction due to mixed up ANSI escape codes. It is idiomatic in such cases to:
- </p>
- <ul>
- <li>
- Disable ANSI coloring for the server via <code>-D<b>NLPCRAFT_ANSI_COLOR_DISABLED</b>=true</code>
- </li>
- <li>
- Pipe out the server log into a dedicated file using Maven plugins like
- <a target="github" href="https://github.com/bazaarvoice/maven-process-plugin"><code>process-exec-maven-plugin</code></a>.
- </li>
- </ul>
- </div>
- <br/>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#server">REST Server</a></li>
- <li><a href="#probe">Data Probe</a></li>
- <li><a href="#config">Configuration</a></li>
- <li><a href="#testing">CI Testing</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
diff --git a/tools/embedded_probe.html b/tools/embedded_probe.html
deleted file mode 100644
index b1a30a9..0000000
--- a/tools/embedded_probe.html
+++ /dev/null
@@ -1,159 +0,0 @@
----
-active_crumb: Embedded Probe
-layout: documentation
-id: embedded_probe
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Embedded probe allows to run data probe in the same JVM process as JVM-based client application.
- </p>
- <p>
- Typically, data probes are launched in their own independent JVMs. However, when a client application is
- itself a JVM-based process, it is sometimes preferable for performance reasons to host
- a data model - and hence the data probe - in the same "client" JVM.
- </p>
- <p>
- In a standard processing flow, when data probe (with its deployed data models) is hosted in its own independent JVM,
- the request from
- the client application takes 4 network hops to go server, then to the data probe and back to the client application (see fig 1. below):
- </p>
- <figure>
- <img class="img-fluid" style="max-width: 500px !important;" src="/images/emb_probe1.png" alt="">
- <figcaption><b>Fig 1.</b> Standard Processing Flow</figcaption>
- </figure>
- <p>
- When performing model unit testing or when client "application" can be combined with the data probe in a single JVM, the
- embedded probe mode provides significant performance advantage. Using embedded probe you can reduce number
- of network hops to 2 in a similar scenario - in which case the <b>hop<sub>3</sub></b> and
- <b>hop<sub>4</sub></b> (see fig. 1) are performed inside of JVM between client and embedded probe:
- </p>
- <figure>
- <img class="img-fluid" style="max-width: 500px !important;" src="/images/emb_probe2.png" alt="">
- <figcaption><b>Fig 2.</b> Embedded Processing Flow</figcaption>
- </figure>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Embedded probe support consists of the following types in <code>org.apache.nlpcraft.probe.embedded</code> package:
- </p>
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/embedded/NCEmbeddedProbe.html">
- NCEmbeddedProbe
- </a> - Embedded probe main class.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/embedded/NCEmbeddedResult.html">
- NCEmbeddedResult
- </a> - Processing result container since result is delivered as a local JVM call.
- </li>
- </ul>
- <p>
- Here's Java code snippet from <a href="/examples/alarm_clock.html">Alarm Clock</a>
- example illustrating the usage of embedded probe together with NLPCraft test framework and JUnit 5:
- </p>
- <pre class="brush: java, highlight: [6, 18]">
-public class AlarmTest {
- private NCTestClient cli;
-
- @BeforeEach
- void setUp() throws NCException, IOException {
- if (NCEmbeddedProbe.start(AlarmModel.class)) {
- cli = new NCTestClientBuilder().newBuilder().build();
-
- cli.open("nlpcraft.alarm.ex");
- }
- }
-
- @AfterEach
- void tearDown() throws NCException, IOException {
- if (cli != null)
- cli.close();
-
- NCEmbeddedProbe.stop();
- }
-
- @Test
- public void test() throws NCException, IOException {
- // Should be passed.
- assertTrue(cli.ask("Ping me in 3 minutes").isOk());
- assertTrue(cli.ask("Buzz me in an hour and 15mins").isOk());
- assertTrue(cli.ask("Set my alarm for 30s").isOk());
- }
-}
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Lines 6 and 18 show the embedded data probe start and stop as it's used in the unit tests.
- </li>
- </ul>
- <p>
- It's important to note that there are number of inherent limitations associated with embedded probe:
- </p>
- <ul>
- <li>
- Embedded probe is only available for JVM-based applications (and can be used with any JVM languages).
- </li>
- <li>
- There can be only one embedded probe per JVM.
- </li>
- <li>
- Once data probe is stopped and cannot be re-started again in the same JVM.
- </li>
- <li>
- Even though the caller can register local-JVM listener for the query results, these results
- will still be asynchronously delivered to the REST server in the usual manner so that other clients
- could fetch these results to maintain internal logging, tracing and metrics. If the client
- application hosting data model and its probe <i>is the only client</i> for that model it needs to cancel the
- request on the REST server after receiving a local-JVM callback to release associated
- resources on the REST server.
- </li>
- </ul>
- <p>
- Note that embedded probe is automatically used by the <a href="/tools/test_framework.html#autotest">auto model validator</a>.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/script.html b/tools/script.html
deleted file mode 100644
index 30e85e3..0000000
--- a/tools/script.html
+++ /dev/null
@@ -1,280 +0,0 @@
----
-active_crumb: Command Line
-layout: documentation
-fa_icon: fa-tools
-id: script
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="tldr">
- <h2 class="section-title">TL;DR; <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Watch this quick video (12:05) for introduction to NLPCraft CLI:
- </p>
- <div>
- <iframe
- width="514"
- height="289"
- src="https://www.youtube.com/embed/LNfny60bmgg?modestbranding=1"
- title="NLPCraft - Command Line Utility"
- frameborder="0"
- allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
- allowfullscreen>
- </iframe>
- </div>
- </section>
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft CLI <code>bin/nlpcraft.{sh|cmd}</code> script is a <b>central management utility</b> for NLPCraft that supports
- interactive REPL and command line modes. Most of the functions this script does can be done through running
- Java applications or using REST API - yet using this script you can perform the same tasks much simpler and quicker.
- From starting and managing REST server and probes, running various built-in tools, to testing models via REST API -
- NLPCraft CLI provides a single centralized utility that makes a quick work out
- of all these routine tasks.
- </p>
- <p>
- The script is only available in either official Apache <a href="/download.html#src">source ZIP</a> or in
- <a href="/download.html#zip">binary ZIP</a> distributions. If you used Maven or similar build tools to
- add NLPCraft to your project as a dependency, this script will not be available out-of-the-box - you will
- need to download ZIP distribution separately.
- </p>
- <div class="bq warn">
- <b>ZIP Distribution</b>
- <p>
- The NLPCraft CLI is only available in <a href="/download.html">ZIP distributions</a>. For Maven/Gradle/SBT
- projects you will need to download <a href="/download.html">ZIP distribution</a> separately.
- </p>
- </div>
- <p>
- The script is located in <code>/bin</code> sub-folder of the installation root and is available as
- <code>nlpcraft.sh</code> for <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i> environments. Both versions provide identical functionality.
- </p>
- </section>
- <section id="quick_start">
- <h2 class="section-title">Quick Start <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- The <code>bin/nlpcraft.{sh|cmd}</code> works in two modes:
- </p>
- <ul>
- <li>
- Command line mode where it executes a single command at a time.
- </li>
- <li>
- Interactive REPL mode. To enter REPL mode run <code>bin/nlpcraft.{sh|cmd}</code> without arguments.
- </li>
- </ul>
- <p>
- All the functionality and command syntax remain the same regardless of whether a command is executed from a command line or
- in REPL mode.
- </p>
- <p>
- To get started, run <code>bin/nlpcraft.{sh|cmd}</code>:
- </p>
- <p>
- <img class="img-fluid" src="/images/cli1.png" alt="">
- </p>
- <p>
- Script starts in the REPL mode by default.
- </p>
- <div class="bq info">
- <b>REPL Prompt</b>
- <p>
- REPL prompt shows 4 pieces of information:
- </p>
- <ul>
- <li>REST server status (<code>ON</code> if server is started and detected, <code>OFF</code> otherwise).</li>
- <li>Local probe status (<code>ON</code> if probe is started and connected to the server, <code>OFF</code> otherwise).</li>
- <li>REST access token if the user is signed in. This token is required for all REST calls.</li>
- <li>Current user working directory.</li>
- </ul>
- <p>
- Note that when using <code>start-server</code> command the <a href="/using-rest.html#users">default user account</a> is automatically signed in
- upon the server start and the access token is obtained. You can sign in with different user account, if necessary.
- </p>
- </div>
- <h2 class="section-sub-title">Get Command List <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Type <code>help</code> to list all supported commands (or run <code>bin/nlpcraft.{sh|cmd} help</code> if in command line mode):
- </p>
- <p>
- <img class="img-fluid" src="/images/cli2.png" alt="">
- </p>
- <p>
- You can get detailed instructions and usage examples for any command
- <code>xxx</code> by typing <code>help --cmd=xxx</code>, for example:
- </p>
- <p>
- <img style="max-width: 914px !important;" class="img-fluid" alt="" src="/images/cli7.png">
- </p>
- <p>
- Use <span class="keyboard">Tab</span> key anytime for auto-completion for commands, parameters, file system paths, and model class names.
- Use <span class="keyboard">↑</span> and <span class="keyboard">↓</span> keys to scroll through command history.
- </p>
- <p>
- To exit <code>bin/nlpcraft.{sh|cmd}</code> script in REPL mode use <code>quit</code> command.
- </p>
- </section>
- <section id="os_commands">
- <h2 class="section-title">OS Commands <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- You can run any external OS-specific commands from <code>bin/nlpcraft.{sh|cmd}</code> script by prepending such
- command with a <code>$</code> symbol. For example:
- </p>
- <pre class="brush: bash">
- > $git pull # Runs 'git pull' in the current directory.
- > $vi /home/myproject/pom.xml # Edit 'pom.xml' using vi.
- > $emacs /home/myproject/MyModel.scala # Edit 'MyModel.scala' using emacs.
- > $ls -la # Run 'ls-la' Unix command.
- > $cmd /c dir # Runs Windows 'dir' command in a separate shell.
- </pre>
- <p>
- Note that <em>stdin</em>, <em>stderr</em> and <em>stdout</em> of the new process will inherit from running <code>bin/nlpcraft.{sh|cmd}</code> script process.
- Running OS commands makes the most sense in REPL mode where you don't want to lose a session context while
- executing external OS commands. Technically, however, external OS commands can be executed from command line mode
- as well.
- </p>
- </section>
- <section id="tips">
- <h2 class="section-title">Tips <span class="amp">&</span> Tricks <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft CLI is standard command line utility that can be used in the familiar ways both in
- Linux/Unix/MacOS and Window environments.
- </p>
- <span class="section-sub-title">Typical Development Workflow <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- Here's the typical workflow of working with NLPCraft using NLPCraft CLI. Note that although all of these
- steps can be performed from IDE or a command line - NLPCraft CLI in REPL model provides the easiest
- way to accomplish these tasks:
- </p>
- <ol>
- <li>
- <code>gen-project</code> command generates new Maven/Gradle/Sbt project for either Java, Scala, Groovy
- or Kotlin that is ready to run.
- </li>
- <li>
- <code>start-server</code> command starts the local REST server. You only need to do it once as it is a
- "fire and forget" component.
- </li>
- <li>
- Modify <span class="amp">&</span> build the project using IDE or editor of your choice. You can
- <a href="#os_commands">launch</a> the editor and build tooling right from NLPCraft CLI.
- </li>
- <li>
- <code>test-model</code> command runs auto-validator for your model automatically starting and stopping an
- embedded data probe. Alternatively, <code>start-probe</code> and <code>ask</code> commands allow to manually
- start the local data probe and issue REST requests.
- <ul>
- <li>
- Note that you can use <code>retest-model</code> and <code>restart-probe</code> commands in REPL mode
- to rerun the latest model test or restart the probe with the last set of parameters accordingly avoiding
- the retyping of all required parameters.
- </li>
- </ul>
- </li>
- <li>
- Repeat steps 3 and 4 as you develop, test and debug your models and applications.
- </li>
- <li>
- Once finished - <code>stop</code> command stops both local REST server and data probe. Note that stopping
- the server once again is an optional step as it is a "fire and forget" component and it can be reused between development
- sessions.
- </li>
- </ol>
- <div class="bq info">
- <p>
- Steps 3 and 4 define a typical development cycle for change, build and test. In most cases you <b>don't need</b>
- to restart the server between multiple sessions.
- </p>
- </div>
- <span class="section-sub-title">Using <code>start-server</code> Command <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- REST server is a fire-and-forget component in NLPCraft. You can start it once and you only need to restart it
- if you change the server configuration. It is customary to add <code>bin/nlpcraft.{sh|cmd} start-server</code>
- call to the OS init script to have it started automatically on boot up.
- </p>
- <span class="section-sub-title">Using Default <code>nlpcraft.conf</code> <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- It is recommended to modify the standard <code>nlpcraft.conf</code> configuration file, i.e. <em>in a standard
- installation location</em>. Commands like <code>start-server</code>, <code>start-probe</code> and
- <code>test-model</code> use this location by default and you will not have to specify alternative
- location or provide other overrides every time you run these commands.
- If you keep all your configuration modifications in the standard <code>nlpcraft.conf</code> file you can simply
- run these commands without any additional arguments - significantly simplifying their usage.
- </p>
- <span class="section-sub-title">Using NLPCraft CLI In Scripts <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></span>
- <p>
- NLPCraft CLI can be used in your own scripts to automate your own specific workflow and tasks.
- All of the commands that utilize REST API produce JSON responses, for a example:
- </p>
- <p>
- <img class="img-fluid" src="/images/cli9.png" alt="">
- </p>
- <p>
- You can parse these JSON responses for your own needs in your scripts. For example, here's the
- code snippet to get the access token from the <code>signin</code> command (on Linux):
- </p>
- <pre class="brush: bash">
- $ bin/nlpcraft.sh no-ansi no-logo signin --email=admin@admin.com --passwd=admin | tail -n +2 | jq -M '.acsTok' | tr -d '"'
- </pre>
- <p>
- ...and the access token is:
- </p>
- <pre class="brush: bash">
- bjok7yraypseyk86KgGae
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- We use <code>no-ansi</code> and <code>no-logo</code> commands that clean up and simplify the
- output so that it would be easier to parse the resulting JSON.
- </li>
- <li>
- We use <code>tail -n +2</code> Unix command to skip the 1st line of the output that contains HTTP return code.
- </li>
- <li>
- We use <a target=_blank href="https://stedolan.github.io/jq/">jq</a> utility to parse JSON inline and extract
- <code>acsTok</code> field.
- </li>
- <li>
- We use <code>tr -d '"'</code> to remove surrounding double quotes to get our final result.
- </li>
- </ul>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#quick_start">Quick Start</a></li>
- <li><a href="#os_commands">OS Commands</a></li>
- <li><a href="#tips">Tips <span class="amp">&</span> Tricks</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/sql_model_gen.html b/tools/sql_model_gen.html
deleted file mode 100644
index 6fa3bc8..0000000
--- a/tools/sql_model_gen.html
+++ /dev/null
@@ -1,272 +0,0 @@
----
-active_crumb: SQL Model Generator
-layout: documentation
-id: sql_model_gen
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When using NLPCraft to develop natural language interface to the existing SQL RDBMS one of the
- routine tasks is to develop a data model that mirrors the SQL schema for the given
- database. Doing it manually can be time consuming and error prone.
- </p>
- <p>
- SQL model generator automates this task. This is a Java-based utility that takes JDBC configuration,
- reads database schema using it and creates initial JSON or YAML stub for the data model. This stub then
- can be further extended and modified.
- </p>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- This utility can be run in several ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh gen-sql --url=jdbc:postgresql://localhost:5432/mydb --driver=org.postgresql.Driver --schema=public --out=model.json
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=gen-sql</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-class" role="tabpanel">
- <pre class="brush: bash">
- $ java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Run this class without arguments to get a full help.
- </li>
- <li>
- Use <code>org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator</code> class directly to execute
- it from IDE or programmatically.
- </li>
- </ul>
- </div>
- </div>
- <h2 class="section-sub-title">Parameters <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- If you run this tool with <code>--help</code> parameter you'll get the full help information for each
- parameter:
- </p>
- <pre class="brush: text">
-NAME:
- NCSqlModelGenerator -- NLPCraft model generator from SQL databases.
-
-SYNOPSIS:
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator [PARAMETERS]
-
-DESCRIPTION:
- This utility generates NLPCraft model stub from a given SQL database schema. You
- can choose database schema, set of tables and columns for which you
- want to generate NLPCraft model. After the model is generated you can
- further configure and customize it for your specific needs.
-
- This Java class can be run from the command line or from an IDE like any other
- Java application. Note that required JDBC driver class must be available on the
- classpath and therefore its JAR should be added to the classpath when running
- this application.
-
-PARAMETERS:
- --url|-r=url
- Mandatory database JDBC URL.
-
- --driver|-d=class
- Mandatory JDBC driver class. Note that 'class' must be a
- fully qualified class name. It should also be available on
- the classpath.
-
- --schema|-s=schema
- Mandatory database schema to scan.
-
- --out|-o=filename
- Mandatory name of the output JSON or YAML model file. It should
- have one of the following extensions: .js, .json, .yml, or .yaml
- File extension determines the output file format.
-
- --user|-u=username
- Optional database user name.
-
- --password|-w=password
- Optional database user password.
-
- --mdlId|-m=id
- Optional generated model ID. By default, the model ID will be 'sql.model.id'.
-
- --mdlVer|-v=version
- Optional generated model version. By default, the model version will be '1.0.0-timestamp'.
-
- --mdlName|-n=name
- Optional generated model name. By default, the model name will be 'SQL-based model'.
-
- --exclude|-e=list
- Optional semicolon-separate list of tables and/or columns to exclude. By
- default, none of the tables and columns in the schema are excluded. See below
- for more information.
-
- --prefix|-f=list
- Optional comma-separate list of table or column name prefixes to remove.
- These prefixes will be removed when name is used for model elements
- synonyms. By default, no prefixes will be removed.
-
- --suffix|-q=list
- Optional comma-separate list of table or column name suffixes to remove.
- These suffixes will be removed when name is used for model elements
- synonyms. By default, no suffixes will be removed.
-
- --include|-i=list
- Optional semicolon-separate list of tables and/or columns to include. By
- default, all tables and columns in the schema are included. See below
- for more information.
-
- --synonyms|-y=true|false
- Optional flag on whether to generated auto synonyms for the model elements.
- Default is true.
-
- --override|-z=true|false
- Optional flag to determine whether to override output file if it already exist.
- If override is disabled (default) and output file exists - a unique file name will
- be used instead.
- Default is false.
-
- --parent|-p=true|false
- Optional flag on whether to use element's parent relationship for
- defining SQL columns and their containing (i.e. parent) tables.
- Default is false.
-
- --help|-h|-?
- Prints this usage information.
-
-DETAILS:
- -r, -d, -s, and -o are mandatory parameters, everything else is optional.
-
- Parameter values can be placed in double (") or single (') quotes which will be
- automatically discarded. Use it to pass strings containing spaces in the command line.
-
- Each -i or -e parameter is a semicolon (;) separated list of table or columns names.
- Each table or column name can be one of following forms:
- - table -- to filter on table names only.
- - table#column -- to filter on both table and column names.
- - #column -- to filter on columns only (regardless of the table).
-
- Table and column names are treated as standard Java regular expressions. Note that
- both '#' and ';' cannot be used inside of the regular expression:
-
- -e="#_.+" -- excludes any columns starting with '_'.
- -e="tmp.+" -- excludes all tables starting with 'tmp'.
- -i="Order.*;#[^_].+" -- includes only tables starting with 'Order' and columns that
- do not start with '_'.
-
-EXAMPLES:
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar org.apache.nlpcraft.model.tools.sqlgen.NCSqlModelGenerator
- -r=jdbc:postgresql://localhost:5432/mydb
- -d=org.postgresql.Driver
- -f="tbl_, col_"
- -q="_tmp, _old, _unused"
- -s=public
- -e="#_.+"
- -o=model.json
- </pre>
- <p>
- After the data model stub is generated:
- </p>
- <ul>
- <li>
- Load generated YAML/JSON-based model using <a target="javadoc" href="https://javadoc.io/static/org.apache.nlpcraft/nlpcraft/0.5.0/org/apache/nlpcraft/model/NCModelFileAdapter.html">NCModelFileAdapter</a>
- class to instantiate model from this file.
- </li>
- <li>
- Modify and extend generated model stub to your own needs. In most cases, you'll need
- to add, remove or modify auto-generated synonyms, add intents, etc. Note, however, that generated model
- is valid and minimally complete and can be used as is.
- </li>
- <li>
- Use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractorBuilder.html">NCSqlSchemaBuilder</a> class to get an object representation of the
- SQL data schema for the model. You can use this object representation along with many utility
- methods in <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractor.html">NCSqlExtractor</a>
- class to efficiently auto-generate SQL queries against the source RDBMS.
- </li>
- </ul>
- <p>
- SQL model generator comes with a several utility classes that can be used to programmatically explore
- generated data model, its elements and their metadata. You should start with the following builders to learn more
- about these utility classes:
- </p>
- <ul>
- <li>
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlExtractorBuilder.html">NCSqlExtractorBuilder</a>
- </li>
- <li>
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/sqlgen/NCSqlSchemaBuilder.html">NCSqlSchemaBuilder</a>
- </li>
- </ul>
- </section>
- <section id="examples">
- <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- SQL model generator is used by <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples/sql">SQL Model</a> example. This example comes with two YAML models:
- </p>
- <ul>
- <li><code>sql_model_init.yaml</code> - original generated model straight out of the SQL model generator.</li>
- <li><code>sql_model.yaml</code> - modified and updated model ultimately used by the example.</li>
- </ul>
- <p>
- It is recommended to run a diff between these two files to see what was actually changed and how.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- <li><a href="#examples">Examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/syn_tool.html b/tools/syn_tool.html
deleted file mode 100644
index 2be6fea..0000000
--- a/tools/syn_tool.html
+++ /dev/null
@@ -1,314 +0,0 @@
----
-active_crumb: Synonyms Tool
-layout: documentation
-id: syn_tool
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Synonym suggester tool takes an existing model, analyses its synonyms and intents and comes up with
- a list of synonyms that are currently missing that you might want to add to your model.
- </p>
- <p>
- This tool is accessed via REST call. It is based on Google's BERT and Facebook fasttext
- models. It requires <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> or
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations present on intent
- callbacks. When invoked, the tool scans the given data model for intents and these annotations, and based on these samples tries to determine
- which synonyms are missing in the model.
- </p>
- <div class="bq info">
- <p>
- <b>Single Word Synonyms</b>
- </p>
- <p>
- Synonym suggester tool analyses only single word synonyms ignoring any multi-word synonyms. You
- can often convert a named element with multi-word synonyms into a combination of multiple named
- elements each with a single word synonyms using <a href="/data-model.html#dsl">Composable NERs</a>
- technique.
- </p>
- </div>
- </section>
- <section id="usage">
- <h2 class="section-title">Usage <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In order to use this tool the <code>ctxword</code> server and NLPCraft server should be started as well as
- the server's configuration should potentially be updated.
- </p>
- <h2 class="section-sub-title"><code>ctxword</code> Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <div class="bq warn">
- <b>Python 3.6-3.8</b>
- <p>
- As of this writing (Dec 2020) the <code>ctxword</code> server and its dependencies work only with Python 3.6-3.8 version.
- </p>
- </div>
- <p>
- 'ctxword' server is a Python-based module that provides BERT and fasttext based implementation
- for finding a contextually related words for a given word from the input sentence. NLPCraft server interacts
- with 'ctxword' server via internal REST interface. To configure NLPCraft server and start 'ctxword' Python-based
- server follow these steps:
- </p>
- <ol>
- <li>
- <p>
- Install necessary dependencies by running the following commands from the NLPCraft installation
- directory:<br/>
- <b>NOTE:</b> this step should only be performed once.
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-nix" role="tab">Linux/Unix/MacOS</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-win" role="tab">Windows</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-nix" role="tabpanel">
- <pre class="brush: bash">
- $ cd nlpcraft/src/main/python/ctxword
- $ bin/install_dependencies.sh
- </pre>
- </div>
- <div class="tab-pane fade show" id="nav-win" role="tabpanel">
- <p></p>
- <p>
- Read <code>src\main\python\ctxword\bin\WINDOWS_SETUP.md</code> file for manual installation instructions.
- </p>
- </div>
- </div>
- </li>
- <li>
- <em>Optional.</em>
- <br/>
- Configure <code>nlpcraft.server.ctxword.url</code> property in <code>nlpcraft.conf</code> file (or your own configuration file).
- This property comes with a default endpoint and you only need to change it if you change the
- 'ctxword' module implementation.
- </li>
- <li>
- Start the 'ctxword' server by running the following commands from NLPCraft installation directory:
- <pre class="brush: bash">
- $ cd nlpcraft/src/main/python/ctxword
- $ bin/start_server.{sh|cmd}
- </pre>
- <div class="bq info">
- <p>
- <b>1st Start</b>
- </p>
- Note that on the first start the server will try to load compressed BERT model which is not yet
- available. It will then download this library and compress it which will take a several minutes
- and may require 10 GB+ of available memory. Subsequent starts will skip this step, and the
- server will start much faster.
- </div>
- </li>
- </ol>
- <h2 class="section-sub-title">REST Server <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST server should be <a href="/server-and-probe.html#server">started</a>.
- </p>
- <h2 class="section-sub-title">Running <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Synonyms tool can be run in two different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-rest" role="tab">REST Call</a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh help --cmd=model-sugsyn
- $ bin/nlpcraft.sh model-sugsyn --mdlId=nlpcraft.alarm.ex --minScore=0.5
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <code>mldId</code> parameter is only required if there is more than one model
- deployed in the connected data probe. If the data probe has only one model you can
- ommit this parameter.
- </li>
- <li>
- <code>minScore</code> - Optional minimum confidence score to include into the result, ranging from 0 to 1, default is 0.
- <code>minScore</code> of 0 will include all results, and <code>minScore</code> of 1 will include only results
- with the absolutely highest confidence score. Values between 0.5 and 0.7 is generally suggested.
- </li>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=model-sugsyn</code> to get a full help on this command.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-rest" role="tabpanel">
- <p></p>
- <p>
- <a href="/using-rest.html">REST API</a> accepts only <code>POST</code> HTTP calls and <code>application/json</code> content type
- for JSON payload and responses. When issuing a REST call for this tool you will be using the following URL:
- </p>
- <pre class="brush: plain">
- https://localhost:8081/api/v1/model/sugsyn
- </pre>
- <p>
- where:
- <dl>
- <dt><code>http</code></dt>
- <dd>Either <code>http</code> or <code>https</code> protocol.</dd>
- <dt><code>localhost:8081</code></dt>
- <dd>Host and port on which REST server is started. <code>localhost:8081</code> is the default configuration and can be <a href="/server-and-probe.html">changed</a>.</dd>
- <dt><code>/api/v1</code></dt>
- <dd>Mandatory prefix indicating API version.</dd>
- <dt><code>model/sugsyn</code></dt>
- <dd>Synonym suggester REST call.</dd>
- </dl>
- <p>
- The parameters should be passed in as JSON:
- </p>
- <pre class="brush: js">
- {
- "acsTok": "qweqw9123uqwe",
- "mdlId": "nlpcraft.alarm.ex",
- "minScore": 0.5
- }
- </pre>
- <p>
- where:
- </p>
- <ul>
- <li>
- <code>acsTok</code> - access token obtain via previous <code>'/signin'</code> call.
- </li>
- <li>
- <code>mdlId</code> - ID of the model to run synonym suggester on.
- </li>
- <li>
- <code>minScore</code> - Optional minimum confidence score to include into the result, ranging from 0 to 1, default is 0.
- <code>minScore</code> of 0 will include all results, and <code>minScore</code> of 1 will include only results
- with the absolutely highest confidence score. Values between 0.5 and 0.7 is generally suggested.
- </li>
- </ul>
- </div>
- </div>
- <p>
- Either way the synonym suggester returns the following JSON result (<code>nlpcraft.alarm.ex</code>
- model from <a href="/examples/alarm_clock.html">Alarm</a> example):
- </p>
- <pre class="brush: js">
-{
-"status": "API_OK",
-"result": {
- "modelId": "nlpcraft.alarm.ex",
- "minScore": 0.5,
- "durationMs": 424.0,
- "timestamp": 1.60091239852E12,
- "suggestions": [
- {
- "x:alarm": [
- {
- "score": 1.0,
- "synonym": "ask"
- },
- {
- "score": 0.9477103542042674,
- "synonym": "join"
- },
- {
- "score": 0.8882341083867801,
- "synonym": "get"
- },
- {
- "score": 0.7330826349218547,
- "synonym": "remember"
- },
- {
- "score": 0.6902880910527778,
- "synonym": "contact"
- },
- {
- "score": 0.6014764219771813,
- "synonym": "time"
- },
- {
- "score": 0.5816398376889104,
- "synonym": "follow"
- },
- {
- "score": 0.5640882890681899,
- "synonym": "watch"
- },
- {
- "score": 0.5139855649326083,
- "synonym": "stop"
- },
- {
- "score": 0.5136895804732818,
- "synonym": "kill"
- },
- {
- "score": 0.5001167992233122,
- "synonym": "send"
- }
- ]
- }
- ],
- "warnings": [
- "Model has too few (3) intents samples. It will negatively affect the quality of suggestions. Try to increase overall sample count to at least 20."
- ]
-}
- </pre>
- <p>
- The result is structured as a list of proposed synonyms with their corresponding scores for each model's
- element.
- You should analyse the results for their fitness for your model and its existing synonyms. The tool cannot guarantee
- that every suggested synonym is appropriate or valid - but it gives a good "courtesy" check for potentially
- missing synonyms.
- </p>
- <div class="bq info">
- <p>
- <b>Run Periodically</b>
- </p>
- <p>
- It is a good idea to run this tool periodically if you are actively changing the model. With dozens or hundreds
- of model elements it is very hard to manually maintain quality set of synonyms. With a good list of
- user input samples for each intent this tool can be indispensable for easy maintenance of the synonyms.
- </p>
- </div>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Usage</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/tools/test_framework.html b/tools/test_framework.html
deleted file mode 100644
index 72d0d4f..0000000
--- a/tools/test_framework.html
+++ /dev/null
@@ -1,324 +0,0 @@
----
-active_crumb: Test Framework
-layout: documentation
-id: test_framework
-fa_icon: fa-tools
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<div class="col-md-8 second-column" xmlns="http://www.w3.org/1999/html">
- <section id="overview">
- <h2 class="section-title">Overview <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft comes built-in with a simple-to-use testing framework that can be used with any
- popular unit testing JVM frameworks such as <a href="https://testng.org">TestNG</a> or
- <a href="https://junit.org">JUnit</a>. Test framework is essentially a simplified implementation
- of the REST client tailor-made for model unit testing.
- </p>
- <p>
- Test framework consists of the following types in <code>org.apache.nlpcraft.model.tools.test</code> package:
- </p>
- <ul>
- <li>
- Auto-testing:
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">
- NCTestAutoModelValidator
- </a> - auto-runner for model tests based on <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a>
- and <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations.
- </li>
- </ul>
- </li>
- <li>
- Using test client directly:
- <ul>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClient.html">
- NCTestClient
- </a> - main model test client interface.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClientBuilder.html">
- NCTestClientBuilder
- </a> - builder for <code>NCTestClient</code> instances.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestResult.html">
- NCTestResult
- </a> - result holder of the test sentence processing.
- </li>
- <li>
- <a
- target="javadoc"
- href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestClientException.html">
- NCTestClientException
- </a> - exception used by the test framework.
- </li>
-
- </ul>
- </ul>
- </section>
- <section id="usage">
- <h2 class="section-title">Test client <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Here's a code for unit test using test client for the model from <a href="/examples/alarm_clock.html">Alarm Clock</a>
- example illustrating the usage of test framework together with JUnit 5:
- </p>
- <pre class="brush: java, highlight: [8, 10, 16, 24, 25, 26]">
-public class AlarmTest {
- private NCTestClient cli;
-
- @BeforeEach
- void setUp() throws NCException, IOException {
- NCEmbeddedProbe.start(AlarmModel.class);
-
- cli = new NCTestClientBuilder().newBuilder().build();
-
- cli.open("nlpcraft.alarm.ex");
- }
-
- @AfterEach
- void tearDown() throws NCException, IOException {
- if (cli != null)
- cli.close();
-
- NCEmbeddedProbe.stop();
- }
-
- @Test
- public void test() throws NCException, IOException {
- // Should be passed.
- assertTrue(cli.ask("Ping me in 3 minutes").isOk());
- assertTrue(cli.ask("Buzz me in an hour and 15mins").isOk());
- assertTrue(cli.ask("Set my alarm for 30s").isOk());
- }
-}
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Before each unit test execution on line 8 we create new instance of <code>NCTestClient</code> using
- client builder with default configuration. On line 10 we open test client connection for model
- with ID <code>nlpcraft.alarm.ex</code>.
- </li>
- <li>
- After each test execution we close test client connection on line 16.
- </li>
- <li>
- On lines 24-26 we submit ("ask") test sentences to our model and check for succesfull processing.
- </li>
- </ul>
- <div class="bq info">
- <p>
- <b>Embedded Probe</b>
- </p>
- <p>
- Note that this example (lines 6 and 18), as all other examples shipped with NLPCraft, uses
- <a href="embedded_probe.html">Embedded Probe</a>.
- </p>
- <p>
- Typically, data probes are launched in their own independent JVMs. However, in some
- cases - e.g. during unit testing - it is more convenient for model implementation or preferable
- for performance reasons to host a data model (and hence the data probe) in the same JVM.
- </p>
- </div>
- </section>
- <section id="autotest">
- <h2 class="section-title">Auto Model Validation <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- The same model from <a href="/examples/alarm_clock.html">Alarm Clock</a> example can be auto validated using
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a>
- class and user defined <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations from the model's callback method:
- </p>
- <pre class="brush: java, highlight: [4, 5, 6]">
-public class AlarmModel extends NCModelFileAdapter {
- @NCIntentRef("alarm")
- @NCIntentSample({
- "Ping me in 3 minutes",
- "Buzz me in an hour and 15mins",
- "Set my alarm for 30s"
- })
- NCResult onMatch(
- NCIntentMatch ctx,
- @NCIntentTerm("nums") List<NCToken> numToks
- ) {
- // ...
- }
-}
- </pre>
- <p>
- Auto model validator takes one or more model IDs (or class names) and performs validation. Validation consists
- of starting an <a href="/tools/embedded_probe.html">embedded probe</a> with a given model, scanning for
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSample.html">@NCIntentSample</a> and
- <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/NCIntentSampleRef.html">@NCIntentSampleRef</a> annotations and their corresponding callback methods, submitting each
- sample input sentences from these annotations and checking that resulting intent matches the intent the sample was attached to.
- </p>
- <h2 class="section-sub-title">Running <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Auto mode validator can be executed in two different ways:
- </p>
- <nav>
- <div class="nav nav-tabs" role="tablist">
- <a class="nav-item nav-link active" data-toggle="tab" href="#nav-script" role="tab">NLPCraft CLI</a>
- <a class="nav-item nav-link" data-toggle="tab" href="#nav-class" role="tab">Java Class <img src="/images/java2-h20.png" alt=""></a>
- </div>
- </nav>
- <div class="tab-content">
- <div class="tab-pane fade show active" id="nav-script" role="tabpanel">
- <pre class="brush: bash">
- $ bin/nlpcraft.sh help --cmd=test-model
- $ bin/nlpcraft.sh test-model --cp=/path/to/my/model/classes
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- <a href="/tools/script.html">NLPCraft CLI</a> is available as <code>nlpcraft.sh</code> for
- <i class="fab fa-fw fa-linux"></i> and <code>nlpcraft.cmd</code>
- for <i class="fab fa-fw fa-windows"></i>.
- </li>
- <li>
- Run <code class="script">bin/nlpcraft.sh help --cmd=test-model</code> to get a full help on this command.
- </li>
- <li>
- In REPL mode you can use <code>retest-model</code> command to re-run the last mode test with the
- same parameters avoiding the need to retype all the parameters again.
- </li>
- </ul>
- </div>
- <div class="tab-pane fade show" id="nav-class" role="tabpanel">
- <pre class="brush: bash">
- java -cp apache-nlpcraft-incubating-{{site.latest_version}}-all-deps.jar:/path/to/my/model/classes -DNLPCRAFT_TEST_MODELS=org.apache.nlpcraft.examples.alarm.AlarmModel org.apache.nlpcraft.model.tools.test.NCTestAutoModelValidator
- </pre>
- <p>
- <b>NOTES:</b>
- </p>
- <ul>
- <li>
- Make sure to add necessary classpath components for the model(s) you want to auto-validate.
- </li>
- <li>
- Specify the following three system property (all optional):
- <table class="gradient-table">
- <thead>
- <tr>
- <th>System Property</th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code>NLPCRAFT_TEST_MODELS</code></td>
- <td>
- <p>
- Optional system property that should contain comma separate
- list of the data model classes to test. If not provided, the models from the probe
- configuration will be used.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-DNLPCRAFT_TEST_MODELS=org.apache.nlpcraft.examples.alarm.AlarmModel</code>
- </p>
- </td>
- </tr>
- <tr>
- <td><code>NLPCRAFT_PROBE_CONFIG</code></td>
- <td>
- <p>
- Optional system property that should contain the path to probe configuration file.
- If not provided, the default probe configuration file will be used.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-NLPCRAFT_PROBE_CONFIG=/opt/my_probe/probe.conf</code>
- </p>
- </td>
- </tr>
- <tr>
- <td><code>NLPCRAFT_TEST_INTENT_IDS</code></td>
- <td>
- <p>
- Optional system property that should contain the comma separated list of
- intent IDs to test. If not provided, all detected intents will be tested.
- </p>
- <p>
- <b>Example:</b><br/>
- <code>-NLPCRAFT_TEST_INTENT_IDS=int1,int2</code>
- </p>
- </td>
- </tr>
- </tbody>
- </table>
- </li>
- <li>
- You can also use <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a> class
- directly to call it programmatically from the code or from IDE.
- </li>
- </ul>
- </div>
- </div>
- <p>
- In the log output you should see the following validation results:
- </p>
- <figure>
- <img class="img-fluid-no-border" src="/images/auto_validation.png" alt="">
- <figcaption><b>Fig 1.</b> Model auto-validation result.</figcaption>
- </figure>
- <p>
- See <a target="javadoc" href="/apis/latest/org/apache/nlpcraft/model/tools/test/NCTestAutoModelValidator.html">NCTestAutoModelValidator</a>
- class for more details.
- </p>
- </section>
- <section id="examples">
- <h2 class="section-title">Examples <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- All <a target="github" href="https://github.com/apache/incubator-nlpcraft/tree/master/nlpcraft-examples">examples</a>
- shipped with NLPCraft come with instructions on how to auto-validate their models.
- </p>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#overview">Overview</a></li>
- <li><a href="#usage">Test client</a></li>
- <li><a href="#autotest">Auto model validation</a></li>
- <li><a href="#examples">Examples</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
-
-
diff --git a/using-rest.html b/using-rest.html
deleted file mode 100644
index 78890a2..0000000
--- a/using-rest.html
+++ /dev/null
@@ -1,232 +0,0 @@
----
-active_crumb: REST API
-layout: documentation
-id: rest
----
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--suppress HtmlDeprecatedTag -->
-<div class="col-md-8 second-column">
- <section>
- <h2 class="section-title">TL;DR; <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- View <a target="swaggerhub" href="https://app.swaggerhub.com/apis-docs/Apache-NLPCraft/apache-nlpcraft/{{site.latest_version}}">Open API Specification</a> in a separate window. 👌
- </p>
- </section>
- <section id="overview">
- <h2 class="section-title">...bit more details <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST API provides a universal integration path for any type of user application to connect to and use NLPCraft.
- By using REST API these applications can ask natural language questions, get results and perform
- many supporting operations such as user management.
- </p>
- <p>
- REST server accepts the REST call,
- interprets and routes it to the appropriate data probe which in turn invokes one of its deployed
- data models. The result of data model processing travels all the way through the same chain of components
- back to the user application that made the original call.
- </p>
- <div class="bq info">
- <p>
- If you downloaded ZIP archive the <a target="swaggerhub" href="https://app.swaggerhub.com/apis-docs/Apache-NLPCraft/apache-nlpcraft/{{site.latest_version}}">REST API specification</a> is also available in
- <code>openapi/nlpcraft_swagger.yml</code> file
- or on <a target="github" href="https://github.com/apache/incubator-nlpcraft/blob/master/openapi/nlpcraft_swagger.yml">GitHub</a>.
- </p>
- </div>
- </section>
- <section id="java-client">
- <h2 class="section-title">Java Client <i class="fab fa-java"></i> <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft provide native Java client that enables easy-to-use Java-based API wrapping standard
- NLPCraft REST APIs. It can be used by any JVM language that provides Java interop such as Scala, Groovy, or Kotlin:
- </p>
- <ul>
- <li>Maven/Grape/Gradle/SBT <a href="download.html#java-client">instructions</a></li>
- <li>Latest <a target=_ href="/apis/java-client/latest/index.html">Javadoc</a></li>
- <li>Watch, star or fork <a target="github" href="https://github.com/apache/incubator-nlpcraft-java-client">GitHub Project</a></li>
- </ul>
-
- </section>
- <section id="format">
- <h2 class="section-title">REST URL <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- REST API accepts only <code>POST</code> HTTP calls and <code>application/json</code> content type
- for JSON payload and responses. When issuing a REST call you will be using the following URL:
- </p>
- <pre class="brush: plain">
- https://localhost:8081/api/v1/signin
- </pre>
- <p>
- where:
- <dl>
- <dt><code>http</code></dt>
- <dd>Either <code>http</code> or <code>https</code> protocol.</dd>
- <dt><code>localhost:8081</code></dt>
- <dd>Host and port on which REST server is started. <code>localhost:8081</code> is the default configuration and can be <a href="/server-and-probe.html">changed</a>.</dd>
- <dt><code>/api/v1</code></dt>
- <dd>Mandatory prefix indicating API version.</dd>
- <dt><code>/signin</code></dt>
- <dd>Specific REST path.</dd>
- </dl>
- <div class="bq info">
- <p>
- <b>Management Script</b>
- </p>
- <p>
- <a href="/tools/script.html"><code>bin/nlpcraft.{sh|cmd}</code></a> script is a <b>central management utility</b> for NLPCraft. Among many
- commands it provides, there are two commands <code>call</code> and <code>rest</code> that allows for simple and
- effective REST API usage against local REST server.
- </p>
- </div>
- </section>
- <section id="users">
- <h2 class="section-title">Users <span class="amp">&</span> Organizations <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- In NLPCraft most of the REST calls require ID of the user. Every user belongs to a company, and each user
- has administrative privileges flag. The notion of the user and company IDs additionally to the model ID provide
- necessary attributes for proper probe authentication, conversation maintenance, and security.
- </p>
- <h2 class="section-sub-title">Default Account <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- NLPCraft comes with the default account:
- </p>
- <ul>
- <li><b>Email: </b>admin@admin.com</li>
- <li><b>Password: </b>admin</li>
- </ul>
- <p>
- This account has administrative privileges and should be used as a bootstrap account to create other accounts.
- Make sure to delete this account when you go to production.
- </p>
- <div class="bq warn">
- <p><b><span style="font-size: 120%; color: #E67E22">âš </span> Default Account <span style="font-size: 120%; color: #E67E22">âš </span></b></p>
- <p>
- Make sure to delete the default account when you go to production or as soon as you have other secure
- administrative account that can act as a bootstrap account.
- </p>
- </div>
- <h2 class="section-sub-title">Admins <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many operations are only available to the users with administrative privileges. Note
- also that some operations will implicitly behave differently based on whether the currently signed in user
- have administrative privileges or not.
- </p>
- <h2 class="section-sub-title">External User ID <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- Many calls on REST API accept external "on-behalf-of" user ID (<code>usrExtId</code>) additionally to the regular
- user ID (<code>usrID</code>). In these methods zero, one or both IDs must be provided. If none are provided
- the ID of the currently signed in user will be used. If both are provided they should point to the same user.
- External user ID allows to use user identification from the external systems without a need to import all the
- existing users into NLPCraft in the first place.
- </p>
- <p>
- Typical usage pattern for integrating NLPCraft into existing
- system is to create a single administrative NLPCraft user, sign in with that account and then use external
- "on-behalf-of" user IDs in all the requests. This way you get the same functionality as if using ordinary
- user IDs but without a need to migrate and synchronize all user accounts from the existing system to NLPCraft.
- </p>
- </section>
- <section id="server_errors">
- <h2 class="section-title">Server Errors <a href="#"><i class="top-link fas fa-fw fa-angle-double-up"></i></a></h2>
- <p>
- When REST server returns HTTP error the response body contains JSON object (<code>Content-Type → application/json</code>)
- with two fields <code>code</code> and <code>msg</code>:
- </p>
- <pre class="brush: js">
- {
- "code": "NC_INVALID_ACCESS_TOKEN",
- "msg": "Unknown access token: PPdxjwXBOIMpAWNgpKq1"
- }
- </pre>
- <p>
- Following tables shows all possible <code>code</code> values for these server errors:
- </p>
- <table class="gradient-table">
- <thead>
- <tr>
- <th>Code</th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code>NC_INVALID_ACCESS_TOKEN</code></td>
- <td>
- The access token is invalid or no longer valid. Note that previously issued existing access
- tokens can expire or be otherwise invalidated and have to be obtained again.
- </td>
- </tr>
- <tr>
- <td><code>NC_SIGNIN_FAILURE</code></td>
- <td>
- Invalid or unknown user email and/or user password during <code>/signin</code> call. This can
- indicate either the attempt to signin by the user that hasn't been added yet, or the invalid
- password for the existing user.
- </td>
- </tr>
- <tr>
- <td><code>NC_NOT_IMPLEMENTED</code></td>
- <td>
- Particular feature is not implemented yet or not available. This is reserved for API extensions and
- future backward compatibility.
- </td>
- </tr>
- <tr>
- <td><code>NC_INVALID_FIELD</code></td>
- <td>
- This indicated a problem with a field. It's either too bit, too small, empty or otherwise invalid.
- See <code>msg</code> field in the error response for details.
- </td>
- </tr>
- <tr>
- <td><code>NC_ADMIN_REQUIRED</code></td>
- <td>
- Calling user is required to have admin privileges to execute given call.
- </td>
- </tr>
- <tr>
- <td><code>NC_INVALID_OPERATION</code></td>
- <td>
- Indicates that a given operation cannot be performed as specified. For example, this error
- is returned when you attempt to remove or revoke admin privileges from the
- last admin user.
- </td>
- </tr>
- <tr>
- <td><code>NC_ERROR</code></td>
- <td>Indicates a general server error.</td>
- </tr>
- </tbody>
- </table>
- </section>
-</div>
-<div class="col-md-2 third-column">
- <ul class="side-nav">
- <li class="side-nav-title">On This Page</li>
- <li><a href="#java-client">Java Client</a></li>
- <li><a href="#format">REST URL</a></li>
- <li><a href="#users">Users <span class="amp">&</span> Org</a></li>
- <li><a href="#server_errors">Server Errors</a></li>
- {% include quick-links.html %}
- </ul>
-</div>
-
-
-
-
