blob: d0eb066fb41a79f532ed87fc301f0aa57e9c8100 [file] [log] [blame]
<!DOCTYPE HTML>
<html lang="" >
<head>
<title>Duplicate Message Detection ยท ActiveMQ Artemis Documentation</title>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<meta name="description" content="">
<meta name="generator" content="GitBook 3.1.1">
<link rel="stylesheet" href="gitbook/style.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-highlight/website.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-search/search.css">
<link rel="stylesheet" href="gitbook/gitbook-plugin-fontsettings/website.css">
<meta name="HandheldFriendly" content="true"/>
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="gitbook/images/apple-touch-icon-precomposed-152.png">
<link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon">
<link rel="next" href="clusters.html" />
<link rel="prev" href="core-bridges.html" />
</head>
<body>
<div class="book">
<div class="book-summary">
<div id="book-search-input" role="search">
<input type="text" placeholder="Type to search" />
</div>
<nav role="navigation">
<ul class="summary">
<li class="chapter " data-level="1.1" data-path="./">
<a href="./">
Introduction
</a>
</li>
<li class="chapter " data-level="1.2" data-path="notice.html">
<a href="notice.html">
Legal Notice
</a>
</li>
<li class="chapter " data-level="1.3" data-path="preface.html">
<a href="preface.html">
Preface
</a>
</li>
<li class="chapter " data-level="1.4" data-path="project-info.html">
<a href="project-info.html">
Project Info
</a>
</li>
<li class="chapter " data-level="1.5" data-path="versions.html">
<a href="versions.html">
Versions
</a>
</li>
<li class="chapter " data-level="1.6" data-path="messaging-concepts.html">
<a href="messaging-concepts.html">
Messaging Concepts
</a>
</li>
<li class="chapter " data-level="1.7" data-path="architecture.html">
<a href="architecture.html">
Architecture
</a>
</li>
<li class="chapter " data-level="1.8" data-path="using-server.html">
<a href="using-server.html">
Using the Server
</a>
</li>
<li class="chapter " data-level="1.9" data-path="upgrading.html">
<a href="upgrading.html">
Upgrading
</a>
</li>
<li class="chapter " data-level="1.10" data-path="address-model.html">
<a href="address-model.html">
Address Model
</a>
</li>
<li class="chapter " data-level="1.11" data-path="protocols-interoperability.html">
<a href="protocols-interoperability.html">
Protocols and Interoperability
</a>
</li>
<li class="chapter " data-level="1.12" data-path="amqp.html">
<a href="amqp.html">
AMQP
</a>
</li>
<li class="chapter " data-level="1.13" data-path="mqtt.html">
<a href="mqtt.html">
MQTT
</a>
</li>
<li class="chapter " data-level="1.14" data-path="stomp.html">
<a href="stomp.html">
STOMP
</a>
</li>
<li class="chapter " data-level="1.15" data-path="openwire.html">
<a href="openwire.html">
OpenWire
</a>
</li>
<li class="chapter " data-level="1.16" data-path="core.html">
<a href="core.html">
Core
</a>
</li>
<li class="chapter " data-level="1.17" data-path="jms-core-mapping.html">
<a href="jms-core-mapping.html">
Mapping JMS Concepts to the Core API
</a>
</li>
<li class="chapter " data-level="1.18" data-path="using-jms.html">
<a href="using-jms.html">
Using JMS
</a>
</li>
<li class="chapter " data-level="1.19" data-path="client-classpath.html">
<a href="client-classpath.html">
The Client Classpath
</a>
</li>
<li class="chapter " data-level="1.20" data-path="examples.html">
<a href="examples.html">
Examples
</a>
</li>
<li class="chapter " data-level="1.21" data-path="wildcard-routing.html">
<a href="wildcard-routing.html">
Routing Messages With Wild Cards
</a>
</li>
<li class="chapter " data-level="1.22" data-path="wildcard-syntax.html">
<a href="wildcard-syntax.html">
Wildcard Syntax
</a>
</li>
<li class="chapter " data-level="1.23" data-path="filter-expressions.html">
<a href="filter-expressions.html">
Filter Expressions
</a>
</li>
<li class="chapter " data-level="1.24" data-path="persistence.html">
<a href="persistence.html">
Persistence
</a>
</li>
<li class="chapter " data-level="1.25" data-path="configuring-transports.html">
<a href="configuring-transports.html">
Configuring Transports
</a>
</li>
<li class="chapter " data-level="1.26" data-path="config-reload.html">
<a href="config-reload.html">
Configuration Reload
</a>
</li>
<li class="chapter " data-level="1.27" data-path="connection-ttl.html">
<a href="connection-ttl.html">
Detecting Dead Connections
</a>
</li>
<li class="chapter " data-level="1.28" data-path="slow-consumers.html">
<a href="slow-consumers.html">
Detecting Slow Consumers
</a>
</li>
<li class="chapter " data-level="1.29" data-path="network-isolation.html">
<a href="network-isolation.html">
Avoiding Network Isolation
</a>
</li>
<li class="chapter " data-level="1.30" data-path="critical-analysis.html">
<a href="critical-analysis.html">
Detecting Broker Issues (Critical Analysis)
</a>
</li>
<li class="chapter " data-level="1.31" data-path="transaction-config.html">
<a href="transaction-config.html">
Resource Manager Configuration
</a>
</li>
<li class="chapter " data-level="1.32" data-path="flow-control.html">
<a href="flow-control.html">
Flow Control
</a>
</li>
<li class="chapter " data-level="1.33" data-path="send-guarantees.html">
<a href="send-guarantees.html">
Guarantees of sends and commits
</a>
</li>
<li class="chapter " data-level="1.34" data-path="undelivered-messages.html">
<a href="undelivered-messages.html">
Message Redelivery and Undelivered Messages
</a>
</li>
<li class="chapter " data-level="1.35" data-path="message-expiry.html">
<a href="message-expiry.html">
Message Expiry
</a>
</li>
<li class="chapter " data-level="1.36" data-path="large-messages.html">
<a href="large-messages.html">
Large Messages
</a>
</li>
<li class="chapter " data-level="1.37" data-path="paging.html">
<a href="paging.html">
Paging
</a>
</li>
<li class="chapter " data-level="1.38" data-path="scheduled-messages.html">
<a href="scheduled-messages.html">
Scheduled Messages
</a>
</li>
<li class="chapter " data-level="1.39" data-path="last-value-queues.html">
<a href="last-value-queues.html">
Last-Value Queues
</a>
</li>
<li class="chapter " data-level="1.40" data-path="ring-queues.html">
<a href="ring-queues.html">
Ring Queues
</a>
</li>
<li class="chapter " data-level="1.41" data-path="exclusive-queues.html">
<a href="exclusive-queues.html">
Exclusive Queues
</a>
</li>
<li class="chapter " data-level="1.42" data-path="message-grouping.html">
<a href="message-grouping.html">
Message Grouping
</a>
</li>
<li class="chapter " data-level="1.43" data-path="consumer-priority.html">
<a href="consumer-priority.html">
Consumer Priority
</a>
</li>
<li class="chapter " data-level="1.44" data-path="pre-acknowledge.html">
<a href="pre-acknowledge.html">
Extra Acknowledge Modes
</a>
</li>
<li class="chapter " data-level="1.45" data-path="management.html">
<a href="management.html">
Management
</a>
</li>
<li class="chapter " data-level="1.46" data-path="management-console.html">
<a href="management-console.html">
Management Console
</a>
</li>
<li class="chapter " data-level="1.47" data-path="metrics.html">
<a href="metrics.html">
Metrics
</a>
</li>
<li class="chapter " data-level="1.48" data-path="security.html">
<a href="security.html">
Security
</a>
</li>
<li class="chapter " data-level="1.49" data-path="masking-passwords.html">
<a href="masking-passwords.html">
Masking Passwords
</a>
</li>
<li class="chapter " data-level="1.50" data-path="broker-plugins.html">
<a href="broker-plugins.html">
Broker Plugins
</a>
</li>
<li class="chapter " data-level="1.51" data-path="resource-limits.html">
<a href="resource-limits.html">
Resource Limits
</a>
</li>
<li class="chapter " data-level="1.52" data-path="jms-bridge.html">
<a href="jms-bridge.html">
The JMS Bridge
</a>
</li>
<li class="chapter " data-level="1.53" data-path="client-reconnection.html">
<a href="client-reconnection.html">
Client Reconnection and Session Reattachment
</a>
</li>
<li class="chapter " data-level="1.54" data-path="diverts.html">
<a href="diverts.html">
Diverting and Splitting Message Flows
</a>
</li>
<li class="chapter " data-level="1.55" data-path="core-bridges.html">
<a href="core-bridges.html">
Core Bridges
</a>
</li>
<li class="chapter active" data-level="1.56" data-path="duplicate-detection.html">
<a href="duplicate-detection.html">
Duplicate Message Detection
</a>
</li>
<li class="chapter " data-level="1.57" data-path="clusters.html">
<a href="clusters.html">
Clusters
</a>
</li>
<li class="chapter " data-level="1.58" data-path="federation.html">
<a href="federation.html">
Federation
</a>
<ul class="articles">
<li class="chapter " data-level="1.58.1" data-path="federation-address.html">
<a href="federation-address.html">
Address Federation
</a>
</li>
<li class="chapter " data-level="1.58.2" data-path="federation-queue.html">
<a href="federation-queue.html">
Queue Federation
</a>
</li>
</ul>
</li>
<li class="chapter " data-level="1.59" data-path="ha.html">
<a href="ha.html">
High Availability and Failover
</a>
</li>
<li class="chapter " data-level="1.60" data-path="graceful-shutdown.html">
<a href="graceful-shutdown.html">
Graceful Server Shutdown
</a>
</li>
<li class="chapter " data-level="1.61" data-path="libaio.html">
<a href="libaio.html">
Libaio Native Libraries
</a>
</li>
<li class="chapter " data-level="1.62" data-path="thread-pooling.html">
<a href="thread-pooling.html">
Thread management
</a>
</li>
<li class="chapter " data-level="1.63" data-path="web-server.html">
<a href="web-server.html">
Embedded Web Server
</a>
</li>
<li class="chapter " data-level="1.64" data-path="logging.html">
<a href="logging.html">
Logging
</a>
</li>
<li class="chapter " data-level="1.65" data-path="rest.html">
<a href="rest.html">
REST Interface
</a>
</li>
<li class="chapter " data-level="1.66" data-path="embedding-activemq.html">
<a href="embedding-activemq.html">
Embedding the Broker
</a>
</li>
<li class="chapter " data-level="1.67" data-path="karaf.html">
<a href="karaf.html">
Apache Karaf
</a>
</li>
<li class="chapter " data-level="1.68" data-path="tomcat.html">
<a href="tomcat.html">
Apache Tomcat
</a>
</li>
<li class="chapter " data-level="1.69" data-path="spring-integration.html">
<a href="spring-integration.html">
Spring Integration
</a>
</li>
<li class="chapter " data-level="1.70" data-path="cdi-integration.html">
<a href="cdi-integration.html">
CDI Integration
</a>
</li>
<li class="chapter " data-level="1.71" data-path="intercepting-operations.html">
<a href="intercepting-operations.html">
Intercepting Operations
</a>
</li>
<li class="chapter " data-level="1.72" data-path="data-tools.html">
<a href="data-tools.html">
Data Tools
</a>
</li>
<li class="chapter " data-level="1.73" data-path="maven-plugin.html">
<a href="maven-plugin.html">
Maven Plugin
</a>
</li>
<li class="chapter " data-level="1.74" data-path="unit-testing.html">
<a href="unit-testing.html">
Unit Testing
</a>
</li>
<li class="chapter " data-level="1.75" data-path="perf-tuning.html">
<a href="perf-tuning.html">
Troubleshooting and Performance Tuning
</a>
</li>
<li class="chapter " data-level="1.76" data-path="configuration-index.html">
<a href="configuration-index.html">
Configuration Reference
</a>
</li>
<li class="divider"></li>
<li>
<a href="https://www.gitbook.com" target="blank" class="gitbook-link">
Published with GitBook
</a>
</li>
</ul>
</nav>
</div>
<div class="book-body">
<div class="body-inner">
<div class="book-header" role="navigation">
<!-- Title -->
<h1>
<i class="fa fa-circle-o-notch fa-spin"></i>
<a href="." >Duplicate Message Detection</a>
</h1>
</div>
<div class="page-wrapper" tabindex="-1" role="main">
<div class="page-inner">
<div id="book-search-results">
<div class="search-noresults">
<section class="normal markdown-section">
<h1 id="duplicate-message-detection">Duplicate Message Detection</h1>
<p>Apache ActiveMQ Artemis includes powerful automatic duplicate message detection,
filtering out duplicate messages without you having to code your own
fiddly duplicate detection logic at the application level. This chapter
will explain what duplicate detection is, how Apache ActiveMQ Artemis uses it and how
and where to configure it.</p>
<p>When sending messages from a client to a server, or indeed from a server
to another server, if the target server or connection fails sometime
after sending the message, but before the sender receives a response
that the send (or commit) was processed successfully then the sender
cannot know for sure if the message was sent successfully to the
address.</p>
<p>If the target server or connection failed after the send was received
and processed but before the response was sent back then the message
will have been sent to the address successfully, but if the target
server or connection failed before the send was received and finished
processing then it will not have been sent to the address successfully.
From the senders point of view it&apos;s not possible to distinguish these
two cases.</p>
<p>When the server recovers this leaves the client in a difficult
situation. It knows the target server failed, but it does not know if
the last message reached its destination ok. If it decides to resend the
last message, then that could result in a duplicate message being sent
to the address. If each message was an order or a trade then this could
result in the order being fulfilled twice or the trade being double
booked. This is clearly not a desirable situation.</p>
<p>Sending the message(s) in a transaction does not help out either. If the
server or connection fails while the transaction commit is being
processed it is also indeterminate whether the transaction was
successfully committed or not!</p>
<p>To solve these issues Apache ActiveMQ Artemis provides automatic duplicate messages
detection for messages sent to addresses.</p>
<h2 id="using-duplicate-detection-for-message-sending">Using Duplicate Detection for Message Sending</h2>
<p>Enabling duplicate message detection for sent messages is simple: you
just need to set a special property on the message to a unique value.
You can create the value however you like, as long as it is unique. When
the target server receives the message it will check if that property is
set, if it is, then it will check in its in memory cache if it has
already received a message with that value of the header. If it has
received a message with the same value before then it will ignore the
message.</p>
<blockquote>
<p><strong>Note:</strong></p>
<p>Using duplicate detection to move messages between nodes can give you
the same <em>once and only once</em> delivery guarantees as if you were using
an XA transaction to consume messages from source and send them to the
target, but with less overhead and much easier configuration than
using XA.</p>
</blockquote>
<p>If you&apos;re sending messages in a transaction then you don&apos;t have to set
the property for <em>every</em> message you send in that transaction, you only
need to set it once in the transaction. If the server detects a
duplicate message for any message in the transaction, then it will
ignore the entire transaction.</p>
<p>The name of the property that you set is given by the value of
<code>org.apache.activemq.artemis.api.core.Message.HDR_DUPLICATE_DETECTION_ID</code>, which
is <code>_AMQ_DUPL_ID</code></p>
<p>The value of the property can be of type <code>byte[]</code> or <code>SimpleString</code> if
you&apos;re using the core API. If you&apos;re using JMS it must be a <code>String</code>,
and its value should be unique. An easy way of generating a unique id is
by generating a UUID.</p>
<p>Here&apos;s an example of setting the property using the core API:</p>
<pre><code class="lang-java">...
ClientMessage message = session.createMessage(<span class="hljs-keyword">true</span>);
SimpleString myUniqueID = <span class="hljs-string">&quot;This is my unique id&quot;</span>; <span class="hljs-comment">// Could use a UUID for this</span>
message.setStringProperty(HDR_DUPLICATE_DETECTION_ID, myUniqueID);
</code></pre>
<p>And here&apos;s an example using the JMS API:</p>
<pre><code class="lang-java">...
Message jmsMessage = session.createMessage();
String myUniqueID = <span class="hljs-string">&quot;This is my unique id&quot;</span>; <span class="hljs-comment">// Could use a UUID for this</span>
message.setStringProperty(HDR_DUPLICATE_DETECTION_ID.toString(), myUniqueID);
...
</code></pre>
<h2 id="configuring-the-duplicate-id-cache">Configuring the Duplicate ID Cache</h2>
<p>The server maintains caches of received values of the
<code>org.apache.activemq.artemis.core.message.impl.HDR_DUPLICATE_DETECTION_ID</code>
property sent to each address. Each address has its own distinct cache.</p>
<p>The cache is a circular fixed size cache. If the cache has a maximum
size of <code>n</code> elements, then the <code>n + 1</code>th id stored will overwrite the
<code>0</code>th element in the cache.</p>
<p>The maximum size of the cache is configured by the parameter
<code>id-cache-size</code> in <code>broker.xml</code>, the default value is
<code>2000</code> elements.</p>
<p>The caches can also be configured to persist to disk or not. This is
configured by the parameter <code>persist-id-cache</code>, also in
<code>broker.xml</code>. If this is set to <code>true</code> then each id will
be persisted to permanent storage as they are received. The default
value for this parameter is <code>true</code>.</p>
<blockquote>
<p><strong>Note:</strong></p>
<p>When choosing a size of the duplicate id cache be sure to set it to a
larger enough size so if you resend messages all the previously sent
ones are in the cache not having been overwritten.</p>
</blockquote>
<h2 id="duplicate-detection-and-bridges">Duplicate Detection and Bridges</h2>
<p>Core bridges can be configured to automatically add a unique duplicate
id value (if there isn&apos;t already one in the message) before forwarding
the message to its target. This ensures that if the target server
crashes or the connection is interrupted and the bridge resends the
message, then if it has already been received by the target server, it
will be ignored.</p>
<p>To configure a core bridge to add the duplicate id header, simply set
the <code>use-duplicate-detection</code> to <code>true</code> when configuring a bridge in
<code>broker.xml</code>.</p>
<p>The default value for this parameter is <code>true</code>.</p>
<p>For more information on core bridges and how to configure them, please
see <a href="core-bridges.html">Core Bridges</a>.</p>
<h2 id="duplicate-detection-and-cluster-connections">Duplicate Detection and Cluster Connections</h2>
<p>Cluster connections internally use core bridges to move messages
reliable between nodes of the cluster. Consequently they can also be
configured to insert the duplicate id header for each message they move
using their internal bridges.</p>
<p>To configure a cluster connection to add the duplicate id header, simply
set the <code>use-duplicate-detection</code> to <code>true</code> when configuring a cluster
connection in <code>broker.xml</code>.</p>
<p>The default value for this parameter is <code>true</code>.</p>
<p>For more information on cluster connections and how to configure them,
please see <a href="clusters.html">Clusters</a>.</p>
</section>
</div>
<div class="search-results">
<div class="has-results">
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
<ul class="search-results-list"></ul>
</div>
<div class="no-results">
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
</div>
</div>
</div>
</div>
</div>
</div>
<a href="core-bridges.html" class="navigation navigation-prev " aria-label="Previous page: Core Bridges">
<i class="fa fa-angle-left"></i>
</a>
<a href="clusters.html" class="navigation navigation-next " aria-label="Next page: Clusters">
<i class="fa fa-angle-right"></i>
</a>
</div>
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"Duplicate Message Detection","level":"1.56","depth":1,"next":{"title":"Clusters","level":"1.57","depth":1,"path":"clusters.md","ref":"clusters.md","articles":[]},"previous":{"title":"Core Bridges","level":"1.55","depth":1,"path":"core-bridges.md","ref":"core-bridges.md","articles":[]},"dir":"ltr"},"config":{"plugins":[],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"github":"apache/activemq-artemis","theme":"default","githubHost":"https://github.com/","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"ActiveMQ Artemis Documentation","links":{"home":"http://activemq.apache.org/artemis","issues":"https://issues.apache.org/jira/browse/ARTEMIS","contribute":"http://activemq.apache.org/contributing.html"},"gitbook":"3.x.x","description":"ActiveMQ Artemis User Guide and Reference Documentation"},"file":{"path":"duplicate-detection.md","mtime":"2019-09-23T17:32:43.905Z","type":"markdown"},"gitbook":{"version":"3.1.1","time":"2019-09-23T17:39:00.612Z"},"basePath":".","book":{"language":""}});
});
</script>
</div>
<script src="gitbook/gitbook.js"></script>
<script src="gitbook/theme.js"></script>
<script src="gitbook/gitbook-plugin-search/search-engine.js"></script>
<script src="gitbook/gitbook-plugin-search/search.js"></script>
<script src="gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
<script src="gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
<script src="gitbook/gitbook-plugin-sharing/buttons.js"></script>
<script src="gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
</body>
</html>