Google Git
Sign in
apache / activemq-website / e4f487fff764263c37e753b31f00d784ed2ba3ad / . / content / components / artemis / documentation / 2.11.0 / connection-ttl.html
blob: 161bba538749e2e8d74eb3ffd1daa107bbbaee64 [file] [log] [blame]
<!DOCTYPE HTML>
<html lang="" >
<head>
<meta charset="UTF-8">
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>Detecting Dead Connections ยท ActiveMQ Artemis Documentation</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="description" content="">
<meta name="generator" content="GitBook 3.2.3">
<link rel="stylesheet" href="gitbook/style.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-highlight/website.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-search/search.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-fontsettings/website.css">
<meta name="HandheldFriendly" content="true"/>
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="gitbook/images/apple-touch-icon-precomposed-152.png">
<link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon">
<link rel="next" href="slow-consumers.html" />
<link rel="prev" href="config-reload.html" />
</head>
<body>
<div class="book">
<div class="book-summary">
<div id="book-search-input" role="search">
<input type="text" placeholder="Type to search" />
</div>
<nav role="navigation">
<ul class="summary">
<li class="chapter " data-level="1.1" data-path="./">
<a href="./">
Introduction
</a>
</li>
<li class="chapter " data-level="1.2" data-path="notice.html">
<a href="notice.html">
Legal Notice
</a>
</li>
<li class="chapter " data-level="1.3" data-path="preface.html">
<a href="preface.html">
Preface
</a>
</li>
<li class="chapter " data-level="1.4" data-path="project-info.html">
<a href="project-info.html">
Project Info
</a>
</li>
<li class="chapter " data-level="1.5" data-path="versions.html">
<a href="versions.html">
Versions
</a>
</li>
<li class="chapter " data-level="1.6" data-path="messaging-concepts.html">
<a href="messaging-concepts.html">
Messaging Concepts
</a>
</li>
<li class="chapter " data-level="1.7" data-path="architecture.html">
<a href="architecture.html">
Architecture
</a>
</li>
<li class="chapter " data-level="1.8" data-path="using-server.html">
<a href="using-server.html">
Using the Server
</a>
</li>
<li class="chapter " data-level="1.9" data-path="upgrading.html">
<a href="upgrading.html">
Upgrading
</a>
</li>
<li class="chapter " data-level="1.10" data-path="address-model.html">
<a href="address-model.html">
Address Model
</a>
</li>
<li class="chapter " data-level="1.11" data-path="protocols-interoperability.html">
<a href="protocols-interoperability.html">
Protocols and Interoperability
</a>
</li>
<li class="chapter " data-level="1.12" data-path="amqp.html">
<a href="amqp.html">
AMQP
</a>
</li>
<li class="chapter " data-level="1.13" data-path="mqtt.html">
<a href="mqtt.html">
MQTT
</a>
</li>
<li class="chapter " data-level="1.14" data-path="stomp.html">
<a href="stomp.html">
STOMP
</a>
</li>
<li class="chapter " data-level="1.15" data-path="openwire.html">
<a href="openwire.html">
OpenWire
</a>
</li>
<li class="chapter " data-level="1.16" data-path="core.html">
<a href="core.html">
Core
</a>
</li>
<li class="chapter " data-level="1.17" data-path="jms-core-mapping.html">
<a href="jms-core-mapping.html">
Mapping JMS Concepts to the Core API
</a>
</li>
<li class="chapter " data-level="1.18" data-path="using-jms.html">
<a href="using-jms.html">
Using JMS
</a>
</li>
<li class="chapter " data-level="1.19" data-path="client-classpath.html">
<a href="client-classpath.html">
The Client Classpath
</a>
</li>
<li class="chapter " data-level="1.20" data-path="examples.html">
<a href="examples.html">
Examples
</a>
</li>
<li class="chapter " data-level="1.21" data-path="wildcard-routing.html">
<a href="wildcard-routing.html">
Routing Messages With Wild Cards
</a>
</li>
<li class="chapter " data-level="1.22" data-path="wildcard-syntax.html">
<a href="wildcard-syntax.html">
Wildcard Syntax
</a>
</li>
<li class="chapter " data-level="1.23" data-path="filter-expressions.html">
<a href="filter-expressions.html">
Filter Expressions
</a>
</li>
<li class="chapter " data-level="1.24" data-path="persistence.html">
<a href="persistence.html">
Persistence
</a>
</li>
<li class="chapter " data-level="1.25" data-path="configuring-transports.html">
<a href="configuring-transports.html">
Configuring Transports
</a>
</li>
<li class="chapter " data-level="1.26" data-path="config-reload.html">
<a href="config-reload.html">
Configuration Reload
</a>
</li>
<li class="chapter active" data-level="1.27" data-path="connection-ttl.html">
<a href="connection-ttl.html">
Detecting Dead Connections
</a>
</li>
<li class="chapter " data-level="1.28" data-path="slow-consumers.html">
<a href="slow-consumers.html">
Detecting Slow Consumers
</a>
</li>
<li class="chapter " data-level="1.29" data-path="network-isolation.html">
<a href="network-isolation.html">
Avoiding Network Isolation
</a>
</li>
<li class="chapter " data-level="1.30" data-path="critical-analysis.html">
<a href="critical-analysis.html">
Detecting Broker Issues (Critical Analysis)
</a>
</li>
<li class="chapter " data-level="1.31" data-path="transaction-config.html">
<a href="transaction-config.html">
Resource Manager Configuration
</a>
</li>
<li class="chapter " data-level="1.32" data-path="flow-control.html">
<a href="flow-control.html">
Flow Control
</a>
</li>
<li class="chapter " data-level="1.33" data-path="send-guarantees.html">
<a href="send-guarantees.html">
Guarantees of sends and commits
</a>
</li>
<li class="chapter " data-level="1.34" data-path="undelivered-messages.html">
<a href="undelivered-messages.html">
Message Redelivery and Undelivered Messages
</a>
</li>
<li class="chapter " data-level="1.35" data-path="message-expiry.html">
<a href="message-expiry.html">
Message Expiry
</a>
</li>
<li class="chapter " data-level="1.36" data-path="large-messages.html">
<a href="large-messages.html">
Large Messages
</a>
</li>
<li class="chapter " data-level="1.37" data-path="paging.html">
<a href="paging.html">
Paging
</a>
</li>
<li class="chapter " data-level="1.38" data-path="scheduled-messages.html">
<a href="scheduled-messages.html">
Scheduled Messages
</a>
</li>
<li class="chapter " data-level="1.39" data-path="last-value-queues.html">
<a href="last-value-queues.html">
Last-Value Queues
</a>
</li>
<li class="chapter " data-level="1.40" data-path="ring-queues.html">
<a href="ring-queues.html">
Ring Queues
</a>
</li>
<li class="chapter " data-level="1.41" data-path="retroactive-addresses.html">
<a href="retroactive-addresses.html">
Retroactive Addresses
</a>
</li>
<li class="chapter " data-level="1.42" data-path="exclusive-queues.html">
<a href="exclusive-queues.html">
Exclusive Queues
</a>
</li>
<li class="chapter " data-level="1.43" data-path="message-grouping.html">
<a href="message-grouping.html">
Message Grouping
</a>
</li>
<li class="chapter " data-level="1.44" data-path="consumer-priority.html">
<a href="consumer-priority.html">
Consumer Priority
</a>
</li>
<li class="chapter " data-level="1.45" data-path="pre-acknowledge.html">
<a href="pre-acknowledge.html">
Extra Acknowledge Modes
</a>
</li>
<li class="chapter " data-level="1.46" data-path="management.html">
<a href="management.html">
Management
</a>
</li>
<li class="chapter " data-level="1.47" data-path="management-console.html">
<a href="management-console.html">
Management Console
</a>
</li>
<li class="chapter " data-level="1.48" data-path="metrics.html">
<a href="metrics.html">
Metrics
</a>
</li>
<li class="chapter " data-level="1.49" data-path="security.html">
<a href="security.html">
Security
</a>
</li>
<li class="chapter " data-level="1.50" data-path="masking-passwords.html">
<a href="masking-passwords.html">
Masking Passwords
</a>
</li>
<li class="chapter " data-level="1.51" data-path="broker-plugins.html">
<a href="broker-plugins.html">
Broker Plugins
</a>
</li>
<li class="chapter " data-level="1.52" data-path="resource-limits.html">
<a href="resource-limits.html">
Resource Limits
</a>
</li>
<li class="chapter " data-level="1.53" data-path="jms-bridge.html">
<a href="jms-bridge.html">
The JMS Bridge
</a>
</li>
<li class="chapter " data-level="1.54" data-path="client-reconnection.html">
<a href="client-reconnection.html">
Client Reconnection and Session Reattachment
</a>
</li>
<li class="chapter " data-level="1.55" data-path="diverts.html">
<a href="diverts.html">
Diverting and Splitting Message Flows
</a>
</li>
<li class="chapter " data-level="1.56" data-path="core-bridges.html">
<a href="core-bridges.html">
Core Bridges
</a>
</li>
<li class="chapter " data-level="1.57" data-path="transformers.html">
<a href="transformers.html">
Transformers
</a>
</li>
<li class="chapter " data-level="1.58" data-path="duplicate-detection.html">
<a href="duplicate-detection.html">
Duplicate Message Detection
</a>
</li>
<li class="chapter " data-level="1.59" data-path="clusters.html">
<a href="clusters.html">
Clusters
</a>
</li>
<li class="chapter " data-level="1.60" data-path="federation.html">
<a href="federation.html">
Federation
</a>
<ul class="articles">
<li class="chapter " data-level="1.60.1" data-path="federation-address.html">
<a href="federation-address.html">
Address Federation
</a>
</li>
<li class="chapter " data-level="1.60.2" data-path="federation-queue.html">
<a href="federation-queue.html">
Queue Federation
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.61" data-path="ha.html">
<a href="ha.html">
High Availability and Failover
</a>
</li>
<li class="chapter " data-level="1.62" data-path="graceful-shutdown.html">
<a href="graceful-shutdown.html">
Graceful Server Shutdown
</a>
</li>
<li class="chapter " data-level="1.63" data-path="libaio.html">
<a href="libaio.html">
Libaio Native Libraries
</a>
</li>
<li class="chapter " data-level="1.64" data-path="thread-pooling.html">
<a href="thread-pooling.html">
Thread management
</a>
</li>
<li class="chapter " data-level="1.65" data-path="web-server.html">
<a href="web-server.html">
Embedded Web Server
</a>
</li>
<li class="chapter " data-level="1.66" data-path="logging.html">
<a href="logging.html">
Logging
</a>
</li>
<li class="chapter " data-level="1.67" data-path="rest.html">
<a href="rest.html">
REST Interface
</a>
</li>
<li class="chapter " data-level="1.68" data-path="embedding-activemq.html">
<a href="embedding-activemq.html">
Embedding the Broker
</a>
</li>
<li class="chapter " data-level="1.69" data-path="karaf.html">
<a href="karaf.html">
Apache Karaf
</a>
</li>
<li class="chapter " data-level="1.70" data-path="tomcat.html">
<a href="tomcat.html">
Apache Tomcat
</a>
</li>
<li class="chapter " data-level="1.71" data-path="spring-integration.html">
<a href="spring-integration.html">
Spring Integration
</a>
</li>
<li class="chapter " data-level="1.72" data-path="cdi-integration.html">
<a href="cdi-integration.html">
CDI Integration
</a>
</li>
<li class="chapter " data-level="1.73" data-path="intercepting-operations.html">
<a href="intercepting-operations.html">
Intercepting Operations
</a>
</li>
<li class="chapter " data-level="1.74" data-path="data-tools.html">
<a href="data-tools.html">
Data Tools
</a>
</li>
<li class="chapter " data-level="1.75" data-path="maven-plugin.html">
<a href="maven-plugin.html">
Maven Plugin
</a>
</li>
<li class="chapter " data-level="1.76" data-path="unit-testing.html">
<a href="unit-testing.html">
Unit Testing
</a>
</li>
<li class="chapter " data-level="1.77" data-path="perf-tuning.html">
<a href="perf-tuning.html">
Troubleshooting and Performance Tuning
</a>
</li>
<li class="chapter " data-level="1.78" data-path="configuration-index.html">
<a href="configuration-index.html">
Configuration Reference
</a>
</li>
<li class="divider"></li>
<li>
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
Published with GitBook
</a>
</li>
</ul>
</nav>
</div>
<div class="book-body">
<div class="body-inner">
<div class="book-header" role="navigation">
<!-- Title -->
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i>
<a href="." >Detecting Dead Connections</a>
</h1>
</div>
<div class="page-wrapper" tabindex="-1" role="main">
<div class="page-inner">
<div id="book-search-results">
<div class="search-noresults">
<section class="normal markdown-section">
<h1 id="detecting-dead-connections">Detecting Dead Connections</h1>
<p>In this section we will discuss connection time-to-live (TTL) and
explain how Apache ActiveMQ Artemis deals with crashed clients and clients which have
exited without cleanly closing their resources.</p>
<h2 id="cleaning-up-resources-on-the-server">Cleaning up Resources on the Server</h2>
<p>Before an Apache ActiveMQ Artemis client application exits it is considered good
practice that it should close its resources in a controlled manner,
using a <code>finally</code> block.</p>
<p>Here&apos;s an example of a well behaved core client application closing its
session and session factory in a finally block:</p>
<pre><code class="lang-java">ServerLocator locator = <span class="hljs-keyword">null</span>;
ClientSessionFactory sf = <span class="hljs-keyword">null</span>;
ClientSession session = <span class="hljs-keyword">null</span>;
<span class="hljs-keyword">try</span> {
locator = ActiveMQClient.createServerLocatorWithoutHA(..);
sf = locator.createClientSessionFactory();;
session = sf.createSession(...);
... do some stuff with the session...
} <span class="hljs-keyword">finally</span> {
<span class="hljs-keyword">if</span> (session != <span class="hljs-keyword">null</span>) {
session.close();
}
<span class="hljs-keyword">if</span> (sf != <span class="hljs-keyword">null</span>) {
sf.close();
}
<span class="hljs-keyword">if</span>(locator != <span class="hljs-keyword">null</span>) {
locator.close();
}
}
</code></pre>
<p>And here&apos;s an example of a well behaved JMS client application:</p>
<pre><code class="lang-java">Connection jmsConnection = <span class="hljs-keyword">null</span>;
<span class="hljs-keyword">try</span> {
ConnectionFactory jmsConnectionFactory = <span class="hljs-keyword">new</span> ActiveMQConnectionFactory(<span class="hljs-string">&quot;tcp://localhost:61616&quot;</span>);
jmsConnection = jmsConnectionFactory.createConnection();
... do some stuff with the connection...
} <span class="hljs-keyword">finally</span> {
<span class="hljs-keyword">if</span> (connection != <span class="hljs-keyword">null</span>) {
connection.close();
}
}
</code></pre>
<p>Or with using auto-closeable feature from Java, which can save a few lines of code:</p>
<pre><code class="lang-java"><span class="hljs-keyword">try</span> (
ActiveMQConnectionFactory jmsConnectionFactory = <span class="hljs-keyword">new</span> ActiveMQConnectionFactory(<span class="hljs-string">&quot;tcp://localhost:61616&quot;</span>);
Connection jmsConnection = jmsConnectionFactory.createConnection()) {
... do some stuff with the connection...
}
</code></pre>
<p>Unfortunately users don&apos;t always write well behaved applications, and
sometimes clients just crash so they don&apos;t have a chance to clean up
their resources!</p>
<p>If this occurs then it can leave server side resources, like sessions,
hanging on the server. If these were not removed they would cause a
resource leak on the server and over time this result in the server
running out of memory or other resources.</p>
<p>We have to balance the requirement for cleaning up dead client resources
with the fact that sometimes the network between the client and the
server can fail and then come back, allowing the client to reconnect.
Apache ActiveMQ Artemis supports client reconnection, so we don&apos;t want to clean up
&quot;dead&quot; server side resources too soon or this will prevent any client
from reconnecting, as it won&apos;t be able to find its old sessions on the
server.</p>
<p>Apache ActiveMQ Artemis makes all of this configurable via a <em>connection TTL</em>.
Basically, the TTL determines how long the server will keep a connection
alive in the absence of any data arriving from the client. The client will
automatically send &quot;ping&quot; packets periodically to prevent the server from
closing it down. If the server doesn&apos;t receive any packets on a connection
for the connection TTL time, then it will automatically close all the
sessions on the server that relate to that connection.</p>
<p>The connection TTL is configured on the URI using the <code>connectionTtl</code>
parameter.</p>
<p>The default value for connection ttl on an &quot;unreliable&quot; connection (e.g.
a Netty connection using the <code>tcp</code> URL scheme) is <code>60000</code>ms, i.e. 1 minute.
The default value for connection ttl on a &quot;reliable&quot; connection (e.g. an
in-vm connection using the <code>vm</code> URL scheme) is <code>-1</code>. A value of <code>-1</code> for
<code>connectionTTL</code> means the server will never time out the connection on
the server side.</p>
<p>If you do not wish clients to be able to specify their own connection
TTL, you can override all values used by a global value set on the
server side. This can be done by specifying the
<code>connection-ttl-override</code> attribute in the server side configuration.
The default value for <code>connection-ttl-override</code> is <code>-1</code> which means &quot;do
not override&quot; (i.e. let clients use their own values).</p>
<p>The logic to check connections for TTL violations runs periodically on
the broker. By default, the checks are done every 2,000 milliseconds.
However, this can be changed if necessary by using the
<code>connection-ttl-check-interval</code> attribute.</p>
<h2 id="closing-forgotten-resources">Closing Forgotten Resources</h2>
<p>As previously discussed, it&apos;s important that all core client sessions
and JMS connections are always closed explicitly in a <code>finally</code> block
when you are finished using them.</p>
<p>If you fail to do so, Apache ActiveMQ Artemis will detect this at garbage collection
time, and log a warning (If you are using JMS the warning will involve a JMS connection).</p>
<p>Apache ActiveMQ Artemis will then close the connection / client session for you.</p>
<p>Note that the log will also tell you the exact line of your user code
where you created the JMS connection / client session that you later did
not close. This will enable you to pinpoint the error in your code and
correct it appropriately.</p>
<h2 id="detecting-failure-from-the-client">Detecting Failure from the Client</h2>
<p>In the previous section we discussed how the client sends pings to the
server and how &quot;dead&quot; connection resources are cleaned up by the server.
There&apos;s also another reason for pinging, and that&apos;s for the <em>client</em> to
be able to detect that the server or network has failed.</p>
<p>As long as the client is receiving data from the server it will consider
the connection to be still alive.</p>
<p>If the client does not receive any packets for a configurable number
of milliseconds then it will consider the connection failed and will
either initiate failover, or call any <code>FailureListener</code> instances (or
<code>ExceptionListener</code> instances if you are using JMS) depending on how
it has been configured.</p>
<p>This is controlled by setting the <code>clientFailureCheckPeriod</code> parameter
on the URI your client is using to connect, e.g.
<code>tcp://localhost:61616?clientFailureCheckPeriod=30000</code>.</p>
<p>The default value for client failure check period on an &quot;unreliable&quot;
connection (e.g. a Netty connection) is <code>30000</code> ms, i.e. 30 seconds. The
default value for client failure check period on a &quot;reliable&quot; connection
(e.g. an in-vm connection) is <code>-1</code>. A value of <code>-1</code> means the client
will never fail the connection on the client side if no data is received
from the server. Typically this is much lower than connection TTL to
allow clients to reconnect in case of transitory failure.</p>
<h2 id="configuring-asynchronous-connection-execution">Configuring Asynchronous Connection Execution</h2>
<p>Most packets received on the server side are executed on the remoting
thread. These packets represent short-running operations and are always
executed on the remoting thread for performance reasons.</p>
<p>However, by default some kinds of packets are executed using a thread
from a thread pool so that the remoting thread is not tied up for too
long. Please note that processing operations asynchronously on another
thread adds a little more latency. These packets are:</p>
<ul>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.RollbackMessage</code></p>
</li>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.SessionCloseMessage</code></p>
</li>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.SessionCommitMessage</code></p>
</li>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.SessionXACommitMessage</code></p>
</li>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.SessionXAPrepareMessage</code></p>
</li>
<li><p><code>org.apache.activemq.artemis.core.protocol.core.impl.wireformat.SessionXARollbackMessage</code></p>
</li>
</ul>
<p>To disable asynchronous connection execution, set the parameter
<code>async-connection-execution-enabled</code> in <code>broker.xml</code> to
<code>false</code> (default value is <code>true</code>).</p>
</section>
</div>
<div class="search-results">
<div class="has-results">
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
<ul class="search-results-list"></ul>
</div>
<div class="no-results">
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
</div>
</div>
</div>
</div>
</div>
</div>
<a href="config-reload.html" class="navigation navigation-prev " aria-label="Previous page: Configuration Reload">
<i class="fa fa-angle-left"></i>
</a>
<a href="slow-consumers.html" class="navigation navigation-next " aria-label="Next page: Detecting Slow Consumers">
<i class="fa fa-angle-right"></i>
</a>
</div>
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"Detecting Dead Connections","level":"1.27","depth":1,"next":{"title":"Detecting Slow Consumers","level":"1.28","depth":1,"path":"slow-consumers.md","ref":"slow-consumers.md","articles":[]},"previous":{"title":"Configuration Reload","level":"1.26","depth":1,"path":"config-reload.md","ref":"config-reload.md","articles":[]},"dir":"ltr"},"config":{"plugins":[],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"github":"apache/activemq-artemis","theme":"default","githubHost":"https://github.com/","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"ActiveMQ Artemis Documentation","links":{"home":"http://activemq.apache.org/artemis","issues":"https://issues.apache.org/jira/browse/ARTEMIS","contribute":"http://activemq.apache.org/contributing.html"},"gitbook":"3.x.x","description":"ActiveMQ Artemis User Guide and Reference Documentation"},"file":{"path":"connection-ttl.md","mtime":"2020-01-10T14:13:27.000Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2020-01-10T14:51:01.535Z"},"basePath":".","book":{"language":""}});
});
</script>
</div>
<script src="gitbook/gitbook.js"></script>
<script src="gitbook/theme.js"></script>
<script src="gitbook/gitbook-plugin-search/search-engine.js"></script>
<script src="gitbook/gitbook-plugin-search/search.js"></script>
<script src="gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
<script src="gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
<script src="gitbook/gitbook-plugin-sharing/buttons.js"></script>
<script src="gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
</body>
</html>