docs/0.14.1-incubating/ingestion/stream-push.html - druid-website - Git at Google

 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <meta name="description" content="Apache Druid">
 <meta name="keywords" content="druid,kafka,database,analytics,streaming,real-time,real time,apache,open source">
 <meta name="author" content="Apache Software Foundation">

 <title>Druid | Stream Push</title>

 <link rel="alternate" type="application/atom+xml" href="/feed">
 <link rel="shortcut icon" href="/img/favicon.png">

 <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css" integrity="sha384-fnmOCqbTlWIlj8LyTjo7mOUStjsKC4pOpQbqyi7RrhN7udi9RwhKkMHpvLbHG9Sr" crossorigin="anonymous">

 <link href='//fonts.googleapis.com/css?family=Open+Sans+Condensed:300,700,300italic|Open+Sans:300italic,400italic,600italic,400,300,600,700' rel='stylesheet' type='text/css'>

 <link rel="stylesheet" href="/css/bootstrap-pure.css?v=1.1">
 <link rel="stylesheet" href="/css/base.css?v=1.1">
 <link rel="stylesheet" href="/css/header.css?v=1.1">
 <link rel="stylesheet" href="/css/footer.css?v=1.1">
 <link rel="stylesheet" href="/css/syntax.css?v=1.1">
 <link rel="stylesheet" href="/css/docs.css?v=1.1">

 <script>
   (function() {
     var cx = '000162378814775985090:molvbm0vggm';
     var gcse = document.createElement('script');
     gcse.type = 'text/javascript';
     gcse.async = true;
     gcse.src = (document.location.protocol == 'https:' ? 'https:' : 'http:') +
         '//cse.google.com/cse.js?cx=' + cx;
     var s = document.getElementsByTagName('script')[0];
     s.parentNode.insertBefore(gcse, s);
   })();
 </script>


   </head>

   <body>
     <!-- Start page_header include -->
 <script src="//ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>

 <div class="top-navigator">
   <div class="container">
     <div class="left-cont">
       <a class="logo" href="/"><span class="druid-logo"></span></a>
     </div>
     <div class="right-cont">
       <ul class="links">
         <li class=""><a href="/technology">Technology</a></li>
         <li class=""><a href="/use-cases">Use Cases</a></li>
         <li class=""><a href="/druid-powered">Powered By</a></li>
         <li class=""><a href="/docs/latest/design/">Docs</a></li>
         <li class=""><a href="/community/">Community</a></li>
         <li class="header-dropdown">
           <a>Apache</a>
           <div class="header-dropdown-menu">
             <a href="https://www.apache.org/" target="_blank">Foundation</a>
             <a href="https://www.apache.org/events/current-event" target="_blank">Events</a>
             <a href="https://www.apache.org/licenses/" target="_blank">License</a>
             <a href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a>
             <a href="https://www.apache.org/security/" target="_blank">Security</a>
             <a href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Sponsorship</a>
           </div>
         </li>
         <li class=" button-link"><a href="/downloads.html">Download</a></li>
       </ul>
     </div>
   </div>
   <div class="action-button menu-icon">
     <span class="fa fa-bars"></span> MENU
   </div>
   <div class="action-button menu-icon-close">
     <span class="fa fa-times"></span> MENU
   </div>
 </div>

 <script type="text/javascript">
   var $menu = $('.right-cont');
   var $menuIcon = $('.menu-icon');
   var $menuIconClose = $('.menu-icon-close');

   function showMenu() {
     $menu.fadeIn(100);
     $menuIcon.fadeOut(100);
     $menuIconClose.fadeIn(100);
   }

   $menuIcon.click(showMenu);

   function hideMenu() {
     $menu.fadeOut(100);
     $menuIconClose.fadeOut(100);
     $menuIcon.fadeIn(100);
   }

   $menuIconClose.click(hideMenu);

   $(window).resize(function() {
     if ($(window).width() >= 840) {
       $menu.fadeIn(100);
       $menuIcon.fadeOut(100);
       $menuIconClose.fadeOut(100);
     }
     else {
       $menu.fadeOut(100);
       $menuIcon.fadeIn(100);
       $menuIconClose.fadeOut(100);
     }
   });
 </script>

 <!-- Stop page_header include -->


     <div class="container doc-container">


       <p> Looking for the <a href="/docs/0.19.0/">latest stable documentation</a>?</p>


       <div class="row">
         <div class="col-md-9 doc-content">
           <p>
             <a class="btn btn-default btn-xs visible-xs-inline-block visible-sm-inline-block" href="#toc">Table of Contents</a>
           </p>
           <!--
   ~ Licensed to the Apache Software Foundation (ASF) under one
   ~ or more contributor license agreements.  See the NOTICE file
   ~ distributed with this work for additional information
   ~ regarding copyright ownership.  The ASF licenses this file
   ~ to you under the Apache License, Version 2.0 (the
   ~ "License"); you may not use this file except in compliance
   ~ with the License.  You may obtain a copy of the License at
   ~
   ~   http://www.apache.org/licenses/LICENSE-2.0
   ~
   ~ Unless required by applicable law or agreed to in writing,
   ~ software distributed under the License is distributed on an
   ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   ~ KIND, either express or implied.  See the License for the
   ~ specific language governing permissions and limitations
   ~ under the License.
   -->

 <h1 id="stream-push">Stream Push</h1>

 <p>Apache Druid (incubating) can connect to any streaming data source through
 <a href="https://github.com/druid-io/tranquility/blob/master/README.md">Tranquility</a>, a package for pushing
 streams to Druid in real-time. Druid does not come bundled with Tranquility, and you will have to download the distribution.</p>

 <div class="note info">
 If you've never loaded streaming data into Druid with Tranquility before, we recommend trying out the
 <a href="../tutorials/tutorial-tranquility.html">stream loading tutorial</a> first and then coming back to this page.
 </div>

 <p>Note that with all streaming ingestion options, you must ensure that incoming data is recent
 enough (within a <a href="#segmentgranularity-and-windowperiod">configurable windowPeriod</a> of the current
 time). Older messages will not be processed in real-time. Historical data is best processed with
 <a href="../ingestion/batch-ingestion.html">batch ingestion</a>.</p>

 <h3 id="server">Server</h3>

 <p>Druid can use <a href="https://github.com/druid-io/tranquility/blob/master/docs/server.md">Tranquility Server</a>, which
 lets you send data to Druid without developing a JVM app. You can run Tranquility server colocated with Druid MiddleManagers
 and Historical processes.</p>

 <p>Tranquility server is started by issuing:</p>
 <div class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>bin/tranquility server -configFile &lt;path_to_config_file&gt;/server.json
 </code></pre></div>
 <p>To customize Tranquility Server:</p>

 <ul>
 <li>In <code>server.json</code>, customize the <code>properties</code> and <code>dataSources</code>.</li>
 <li>If you have servers already running Tranquility, stop them (CTRL-C) and start
 them up again.</li>
 </ul>

 <p>For tips on customizing <code>server.json</code>, see the
 <em><a href="../tutorials/tutorial-ingestion-spec.html">Writing an ingestion spec</a></em> tutorial and the
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/server.md">Tranquility Server documentation</a>.</p>

 <h3 id="jvm-apps-and-stream-processors">JVM apps and stream processors</h3>

 <p>Tranquility can also be embedded in JVM-based applications as a library. You can do this directly
 in your own program using the
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/core.md">Core API</a>, or you can use
 the connectors bundled in Tranquility for popular JVM-based stream processors such as
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/storm.md">Storm</a>,
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/samza.md">Samza</a>,
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/spark.md">Spark Streaming</a>, and
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/flink.md">Flink</a>.</p>

 <h3 id="kafka-deprecated">Kafka (Deprecated)</h3>

 <div class="note info">
 NOTE: Tranquility Kafka is deprecated. Please use the <a href="../development/extensions-core/kafka-ingestion.html">Kafka Indexing Service</a> to load data from Kafka instead.
 </div>

 <p><a href="https://github.com/druid-io/tranquility/blob/master/docs/kafka.md">Tranquility Kafka</a>
 lets you load data from Kafka into Druid without writing any code. You only need a configuration
 file.</p>

 <p>Tranquility server is started by issuing:</p>
 <div class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>bin/tranquility kafka -configFile &lt;path_to_config_file&gt;/kafka.json
 </code></pre></div>
 <p>To customize Tranquility Kafka in the single-machine quickstart configuration:</p>

 <ul>
 <li>In <code>kafka.json</code>, customize the <code>properties</code> and <code>dataSources</code>.</li>
 <li>If you have Tranquility already running, stop it (CTRL-C) and start it up again.</li>
 </ul>

 <p>For tips on customizing <code>kafka.json</code>, see the
 <a href="https://github.com/druid-io/tranquility/blob/master/docs/kafka.md">Tranquility Kafka documentation</a>.</p>

 <h2 id="concepts">Concepts</h2>

 <h3 id="task-creation">Task creation</h3>

 <p>Tranquility automates creation of Druid realtime indexing tasks, handling partitioning, replication,
 service discovery, and schema rollover for you, seamlessly and without downtime. You never have to
 write code to deal with individual tasks directly. But, it can be helpful to understand how
 Tranquility creates tasks.</p>

 <p>Tranquility spawns relatively short-lived tasks periodically, and each one handles a small number of
 <a href="../design/segments.html">Druid segments</a>. Tranquility coordinates all task
 creation through ZooKeeper. You can start up as many Tranquility instances as you like with the same
 configuration, even on different machines, and they will send to the same set of tasks.</p>

 <p>See the <a href="https://github.com/druid-io/tranquility/blob/master/docs/overview.md">Tranquility overview</a>
 for more details about how Tranquility manages tasks.</p>

 <h3 id="segmentgranularity-and-windowperiod">segmentGranularity and windowPeriod</h3>

 <p>The segmentGranularity is the time period covered by the segments produced by each task. For
 example, a segmentGranularity of &quot;hour&quot; will spawn tasks that create segments covering one hour
 each.</p>

 <p>The windowPeriod is the slack time permitted for events. For example, a windowPeriod of ten minutes
 (the default) means that any events with a timestamp older than ten minutes in the past, or more
 than ten minutes in the future, will be dropped.</p>

 <p>These are important configurations because they influence how long tasks will be alive for, and how
 long data stays in the realtime system before being handed off to the Historical processes. For example,
 if your configuration has segmentGranularity &quot;hour&quot; and windowPeriod ten minutes, tasks will stay
 around listening for events for an hour and ten minutes. For this reason, to prevent excessive
 buildup of tasks, it is recommended that your windowPeriod be less than your segmentGranularity.</p>

 <h3 id="append-only">Append only</h3>

 <p>Druid streaming ingestion is <em>append-only</em>, meaning you cannot use streaming ingestion to update or
 delete individual records after they are inserted. If you need to update or delete individual
 records, you need to use a batch reindexing process. See the <em><a href="batch-ingestion.html">batch ingest</a></em>
 page for more details.</p>

 <p>Druid does support efficient deletion of entire time ranges without resorting to batch reindexing.
 This can be done automatically through setting up retention policies.</p>

 <h3 id="guarantees">Guarantees</h3>

 <p>Tranquility operates under a best-effort design. It tries reasonably hard to preserve your data, by allowing you to set
 up replicas and by retrying failed pushes for a period of time, but it does not guarantee that your events will be
 processed exactly once. In some conditions, it can drop or duplicate events:</p>

 <ul>
 <li>Events with timestamps outside your configured windowPeriod will be dropped.</li>
 <li>If you suffer more Druid Middle Manager failures than your configured replicas count, some
 partially indexed data may be lost.</li>
 <li>If there is a persistent issue that prevents communication with the Druid indexing service, and
 retry policies are exhausted during that period, or the period lasts longer than your windowPeriod,
 some events will be dropped.</li>
 <li>If there is an issue that prevents Tranquility from receiving an acknowledgement from the indexing
 service, it will retry the batch, which can lead to duplicated events.</li>
 <li>If you are using Tranquility inside Storm or Samza, various parts of both architectures have an
 at-least-once design and can lead to duplicated events.</li>
 </ul>

 <p>Under normal operation, these risks are minimal. But if you need absolute 100% fidelity for
 historical data, we recommend a hybrid/batch streaming architecture, described below.</p>

 <h3 id="hybrid-batch-streaming">Hybrid Batch/Streaming</h3>

 <p>You can combine batch and streaming methods in a hybrid batch/streaming architecture. In a hybrid architecture, you use a streaming method to do initial ingestion, and then periodically re-ingest older data in batch mode (typically every few hours, or nightly). When Druid re-ingests data for a time range, the new data automatically replaces the data from the earlier ingestion.</p>

 <p>All streaming ingestion methods currently supported by Druid do introduce the possibility of dropped or duplicated messages in certain failure scenarios, and batch re-ingestion eliminates this potential source of error for historical data.</p>

 <p>Batch re-ingestion also gives you the option to re-ingest your data if you needed to revise it for any reason.</p>

 <h3 id="deployment-notes">Deployment Notes</h3>

 <p>Stream ingestion may generate a large number of small segments because it&#39;s difficult to optimize the segment size at
 ingestion time. The number of segments will increase over time, and this might cuase the query performance issue. </p>

 <p>Details on how to optimize the segment size can be found on <a href="../operations/segment-optimization.html">Segment size optimization</a>.</p>

 <h2 id="documentation">Documentation</h2>

 <p>Tranquility documentation be found <a href="https://github.com/druid-io/tranquility/blob/master/README.md">here</a>.</p>

 <h2 id="configuration">Configuration</h2>

 <p>Tranquility configuration can be found <a href="https://github.com/druid-io/tranquility/blob/master/docs/configuration.md">here</a>.</p>

 <p>Tranquility&#39;s tuningConfig can be found <a href="http://static.druid.io/tranquility/api/latest/#com.metamx.tranquility.druid.DruidTuning">here</a>. </p>

         </div>
         <div class="col-md-3">
           <div class="searchbox">
             <gcse:searchbox-only></gcse:searchbox-only>
           </div>
           <div id="toc" class="nav toc hidden-print">
           </div>
         </div>
       </div>
     </div>

     <!-- Start page_footer include -->
 <footer class="druid-footer">
 <div class="container">
   <div class="text-center">
     <p>
     <a href="/technology">Technology</a>&ensp;·&ensp;
     <a href="/use-cases">Use Cases</a>&ensp;·&ensp;
     <a href="/druid-powered">Powered by Druid</a>&ensp;·&ensp;
     <a href="/docs/latest/">Docs</a>&ensp;·&ensp;
     <a href="/community/">Community</a>&ensp;·&ensp;
     <a href="/downloads.html">Download</a>&ensp;·&ensp;
     <a href="/faq">FAQ</a>
     </p>
   </div>
   <div class="text-center">
     <a title="Join the user group" href="https://groups.google.com/forum/#!forum/druid-user" target="_blank"><span class="fa fa-comments"></span></a>&ensp;·&ensp;
     <a title="Follow Druid" href="https://twitter.com/druidio" target="_blank"><span class="fab fa-twitter"></span></a>&ensp;·&ensp;
     <a title="GitHub" href="https://github.com/apache/druid" target="_blank"><span class="fab fa-github"></span></a>
   </div>
   <div class="text-center license">
     Copyright © 2020 <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a>.<br>
     Except where otherwise noted, licensed under <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>.<br>
     Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
   </div>
 </div>
 </footer>

 <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script>
 <script>
   window.dataLayer = window.dataLayer || [];
   function gtag(){dataLayer.push(arguments);}
   gtag('js', new Date());
   gtag('config', 'UA-131010415-1');
 </script>
 <script>
   function trackDownload(type, url) {
     ga('send', 'event', 'download', type, url);
   }
 </script>
 <script src="//code.jquery.com/jquery.min.js"></script>
 <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
 <script src="/assets/js/druid.js"></script>
 <!-- stop page_footer include -->


     <script>
     $(function() {
       $(".toc").load("/docs/0.14.1-incubating/toc.html");

       // There is no way to tell when .gsc-input will be async loaded into the page so just try to set a placeholder until it works
       var tries = 0;
       var timer = setInterval(function() {
         tries++;
         if (tries > 300) clearInterval(timer);
         var searchInput = $('input.gsc-input');
         if (searchInput.length) {
           searchInput.attr('placeholder', 'Search');
           clearInterval(timer);
         }
       }, 100);
     });
     </script>
   </body>
 </html>
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<meta name="description" content="Apache Druid">
	<meta name="keywords" content="druid,kafka,database,analytics,streaming,real-time,real time,apache,open source">
	<meta name="author" content="Apache Software Foundation">

	<title>Druid \| Stream Push</title>

	<link rel="alternate" type="application/atom+xml" href="/feed">
	<link rel="shortcut icon" href="/img/favicon.png">

	<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css" integrity="sha384-fnmOCqbTlWIlj8LyTjo7mOUStjsKC4pOpQbqyi7RrhN7udi9RwhKkMHpvLbHG9Sr" crossorigin="anonymous">

	<link href='//fonts.googleapis.com/css?family=Open+Sans+Condensed:300,700,300italic\|Open+Sans:300italic,400italic,600italic,400,300,600,700' rel='stylesheet' type='text/css'>

	<link rel="stylesheet" href="/css/bootstrap-pure.css?v=1.1">
	<link rel="stylesheet" href="/css/base.css?v=1.1">
	<link rel="stylesheet" href="/css/header.css?v=1.1">
	<link rel="stylesheet" href="/css/footer.css?v=1.1">
	<link rel="stylesheet" href="/css/syntax.css?v=1.1">
	<link rel="stylesheet" href="/css/docs.css?v=1.1">

	<script>
	(function() {
	var cx = '000162378814775985090:molvbm0vggm';
	var gcse = document.createElement('script');
	gcse.type = 'text/javascript';
	gcse.async = true;
	gcse.src = (document.location.protocol == 'https:' ? 'https:' : 'http:') +
	'//cse.google.com/cse.js?cx=' + cx;
	var s = document.getElementsByTagName('script')[0];
	s.parentNode.insertBefore(gcse, s);
	})();
	</script>


	</head>

	<body>
	<!-- Start page_header include -->
	<script src="//ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>

	<div class="top-navigator">
	<div class="container">
	<div class="left-cont">
	<a class="logo" href="/"><span class="druid-logo"></span></a>
	</div>
	<div class="right-cont">
	<ul class="links">
	<li class=""><a href="/technology">Technology</a></li>
	<li class=""><a href="/use-cases">Use Cases</a></li>
	<li class=""><a href="/druid-powered">Powered By</a></li>
	<li class=""><a href="/docs/latest/design/">Docs</a></li>
	<li class=""><a href="/community/">Community</a></li>
	<li class="header-dropdown">
	<a>Apache</a>
	<div class="header-dropdown-menu">
	<a href="https://www.apache.org/" target="_blank">Foundation</a>
	<a href="https://www.apache.org/events/current-event" target="_blank">Events</a>
	<a href="https://www.apache.org/licenses/" target="_blank">License</a>
	<a href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a>
	<a href="https://www.apache.org/security/" target="_blank">Security</a>
	<a href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Sponsorship</a>
	</div>
	</li>
	<li class=" button-link"><a href="/downloads.html">Download</a></li>
	</ul>
	</div>
	</div>
	<div class="action-button menu-icon">
	<span class="fa fa-bars"></span> MENU
	</div>
	<div class="action-button menu-icon-close">
	<span class="fa fa-times"></span> MENU
	</div>
	</div>

	<script type="text/javascript">
	var $menu = $('.right-cont');
	var $menuIcon = $('.menu-icon');
	var $menuIconClose = $('.menu-icon-close');

	function showMenu() {
	$menu.fadeIn(100);
	$menuIcon.fadeOut(100);
	$menuIconClose.fadeIn(100);
	}

	$menuIcon.click(showMenu);

	function hideMenu() {
	$menu.fadeOut(100);
	$menuIconClose.fadeOut(100);
	$menuIcon.fadeIn(100);
	}

	$menuIconClose.click(hideMenu);

	$(window).resize(function() {
	if ($(window).width() >= 840) {
	$menu.fadeIn(100);
	$menuIcon.fadeOut(100);
	$menuIconClose.fadeOut(100);
	}
	else {
	$menu.fadeOut(100);
	$menuIcon.fadeIn(100);
	$menuIconClose.fadeOut(100);
	}
	});
	</script>

	<!-- Stop page_header include -->


	<div class="container doc-container">




	<p> Looking for the <a href="/docs/0.19.0/">latest stable documentation</a>?</p>


	<div class="row">
	<div class="col-md-9 doc-content">
	<p>
	<a class="btn btn-default btn-xs visible-xs-inline-block visible-sm-inline-block" href="#toc">Table of Contents</a>
	</p>
	<!--
	~ Licensed to the Apache Software Foundation (ASF) under one
	~ or more contributor license agreements. See the NOTICE file
	~ distributed with this work for additional information
	~ regarding copyright ownership. The ASF licenses this file
	~ to you under the Apache License, Version 2.0 (the
	~ "License"); you may not use this file except in compliance
	~ with the License. You may obtain a copy of the License at
	~
	~ http://www.apache.org/licenses/LICENSE-2.0
	~
	~ Unless required by applicable law or agreed to in writing,
	~ software distributed under the License is distributed on an
	~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	~ KIND, either express or implied. See the License for the
	~ specific language governing permissions and limitations
	~ under the License.
	-->

	<h1 id="stream-push">Stream Push</h1>

	<p>Apache Druid (incubating) can connect to any streaming data source through
	<a href="https://github.com/druid-io/tranquility/blob/master/README.md">Tranquility</a>, a package for pushing
	streams to Druid in real-time. Druid does not come bundled with Tranquility, and you will have to download the distribution.</p>

	<div class="note info">
	If you've never loaded streaming data into Druid with Tranquility before, we recommend trying out the
	<a href="../tutorials/tutorial-tranquility.html">stream loading tutorial</a> first and then coming back to this page.
	</div>

	<p>Note that with all streaming ingestion options, you must ensure that incoming data is recent
	enough (within a <a href="#segmentgranularity-and-windowperiod">configurable windowPeriod</a> of the current
	time). Older messages will not be processed in real-time. Historical data is best processed with
	<a href="../ingestion/batch-ingestion.html">batch ingestion</a>.</p>

	<h3 id="server">Server</h3>

	<p>Druid can use <a href="https://github.com/druid-io/tranquility/blob/master/docs/server.md">Tranquility Server</a>, which
	lets you send data to Druid without developing a JVM app. You can run Tranquility server colocated with Druid MiddleManagers
	and Historical processes.</p>

	<p>Tranquility server is started by issuing:</p>
	<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>bin/tranquility server -configFile <path_to_config_file>/server.json
	</code></pre></div>
	<p>To customize Tranquility Server:</p>

	<ul>
	<li>In <code>server.json</code>, customize the <code>properties</code> and <code>dataSources</code>.</li>
	<li>If you have servers already running Tranquility, stop them (CTRL-C) and start
	them up again.</li>
	</ul>

	<p>For tips on customizing <code>server.json</code>, see the
	<em><a href="../tutorials/tutorial-ingestion-spec.html">Writing an ingestion spec</a></em> tutorial and the
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/server.md">Tranquility Server documentation</a>.</p>

	<h3 id="jvm-apps-and-stream-processors">JVM apps and stream processors</h3>

	<p>Tranquility can also be embedded in JVM-based applications as a library. You can do this directly
	in your own program using the
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/core.md">Core API</a>, or you can use
	the connectors bundled in Tranquility for popular JVM-based stream processors such as
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/storm.md">Storm</a>,
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/samza.md">Samza</a>,
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/spark.md">Spark Streaming</a>, and
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/flink.md">Flink</a>.</p>

	<h3 id="kafka-deprecated">Kafka (Deprecated)</h3>

	<div class="note info">
	NOTE: Tranquility Kafka is deprecated. Please use the <a href="../development/extensions-core/kafka-ingestion.html">Kafka Indexing Service</a> to load data from Kafka instead.
	</div>

	<p><a href="https://github.com/druid-io/tranquility/blob/master/docs/kafka.md">Tranquility Kafka</a>
	lets you load data from Kafka into Druid without writing any code. You only need a configuration
	file.</p>

	<p>Tranquility server is started by issuing:</p>
	<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>bin/tranquility kafka -configFile <path_to_config_file>/kafka.json
	</code></pre></div>
	<p>To customize Tranquility Kafka in the single-machine quickstart configuration:</p>

	<ul>
	<li>In <code>kafka.json</code>, customize the <code>properties</code> and <code>dataSources</code>.</li>
	<li>If you have Tranquility already running, stop it (CTRL-C) and start it up again.</li>
	</ul>

	<p>For tips on customizing <code>kafka.json</code>, see the
	<a href="https://github.com/druid-io/tranquility/blob/master/docs/kafka.md">Tranquility Kafka documentation</a>.</p>

	<h2 id="concepts">Concepts</h2>

	<h3 id="task-creation">Task creation</h3>

	<p>Tranquility automates creation of Druid realtime indexing tasks, handling partitioning, replication,
	service discovery, and schema rollover for you, seamlessly and without downtime. You never have to
	write code to deal with individual tasks directly. But, it can be helpful to understand how
	Tranquility creates tasks.</p>

	<p>Tranquility spawns relatively short-lived tasks periodically, and each one handles a small number of
	<a href="../design/segments.html">Druid segments</a>. Tranquility coordinates all task
	creation through ZooKeeper. You can start up as many Tranquility instances as you like with the same
	configuration, even on different machines, and they will send to the same set of tasks.</p>

	<p>See the <a href="https://github.com/druid-io/tranquility/blob/master/docs/overview.md">Tranquility overview</a>
	for more details about how Tranquility manages tasks.</p>

	<h3 id="segmentgranularity-and-windowperiod">segmentGranularity and windowPeriod</h3>

	<p>The segmentGranularity is the time period covered by the segments produced by each task. For
	example, a segmentGranularity of "hour" will spawn tasks that create segments covering one hour
	each.</p>

	<p>The windowPeriod is the slack time permitted for events. For example, a windowPeriod of ten minutes
	(the default) means that any events with a timestamp older than ten minutes in the past, or more
	than ten minutes in the future, will be dropped.</p>

	<p>These are important configurations because they influence how long tasks will be alive for, and how
	long data stays in the realtime system before being handed off to the Historical processes. For example,
	if your configuration has segmentGranularity "hour" and windowPeriod ten minutes, tasks will stay
	around listening for events for an hour and ten minutes. For this reason, to prevent excessive
	buildup of tasks, it is recommended that your windowPeriod be less than your segmentGranularity.</p>

	<h3 id="append-only">Append only</h3>

	<p>Druid streaming ingestion is <em>append-only</em>, meaning you cannot use streaming ingestion to update or
	delete individual records after they are inserted. If you need to update or delete individual
	records, you need to use a batch reindexing process. See the <em><a href="batch-ingestion.html">batch ingest</a></em>
	page for more details.</p>

	<p>Druid does support efficient deletion of entire time ranges without resorting to batch reindexing.
	This can be done automatically through setting up retention policies.</p>

	<h3 id="guarantees">Guarantees</h3>

	<p>Tranquility operates under a best-effort design. It tries reasonably hard to preserve your data, by allowing you to set
	up replicas and by retrying failed pushes for a period of time, but it does not guarantee that your events will be
	processed exactly once. In some conditions, it can drop or duplicate events:</p>

	<ul>
	<li>Events with timestamps outside your configured windowPeriod will be dropped.</li>
	<li>If you suffer more Druid Middle Manager failures than your configured replicas count, some
	partially indexed data may be lost.</li>
	<li>If there is a persistent issue that prevents communication with the Druid indexing service, and
	retry policies are exhausted during that period, or the period lasts longer than your windowPeriod,
	some events will be dropped.</li>
	<li>If there is an issue that prevents Tranquility from receiving an acknowledgement from the indexing
	service, it will retry the batch, which can lead to duplicated events.</li>
	<li>If you are using Tranquility inside Storm or Samza, various parts of both architectures have an
	at-least-once design and can lead to duplicated events.</li>
	</ul>

	<p>Under normal operation, these risks are minimal. But if you need absolute 100% fidelity for
	historical data, we recommend a hybrid/batch streaming architecture, described below.</p>

	<h3 id="hybrid-batch-streaming">Hybrid Batch/Streaming</h3>

	<p>You can combine batch and streaming methods in a hybrid batch/streaming architecture. In a hybrid architecture, you use a streaming method to do initial ingestion, and then periodically re-ingest older data in batch mode (typically every few hours, or nightly). When Druid re-ingests data for a time range, the new data automatically replaces the data from the earlier ingestion.</p>

	<p>All streaming ingestion methods currently supported by Druid do introduce the possibility of dropped or duplicated messages in certain failure scenarios, and batch re-ingestion eliminates this potential source of error for historical data.</p>

	<p>Batch re-ingestion also gives you the option to re-ingest your data if you needed to revise it for any reason.</p>

	<h3 id="deployment-notes">Deployment Notes</h3>

	<p>Stream ingestion may generate a large number of small segments because it's difficult to optimize the segment size at
	ingestion time. The number of segments will increase over time, and this might cuase the query performance issue. </p>

	<p>Details on how to optimize the segment size can be found on <a href="../operations/segment-optimization.html">Segment size optimization</a>.</p>

	<h2 id="documentation">Documentation</h2>

	<p>Tranquility documentation be found <a href="https://github.com/druid-io/tranquility/blob/master/README.md">here</a>.</p>

	<h2 id="configuration">Configuration</h2>

	<p>Tranquility configuration can be found <a href="https://github.com/druid-io/tranquility/blob/master/docs/configuration.md">here</a>.</p>

	<p>Tranquility's tuningConfig can be found <a href="http://static.druid.io/tranquility/api/latest/#com.metamx.tranquility.druid.DruidTuning">here</a>. </p>

	</div>
	<div class="col-md-3">
	<div class="searchbox">
	<gcse:searchbox-only></gcse:searchbox-only>
	</div>
	<div id="toc" class="nav toc hidden-print">
	</div>
	</div>
	</div>
	</div>

	<!-- Start page_footer include -->
	<footer class="druid-footer">
	<div class="container">
	<div class="text-center">
	<p>
	<a href="/technology">Technology</a>&ensp;·&ensp;
	<a href="/use-cases">Use Cases</a>&ensp;·&ensp;
	<a href="/druid-powered">Powered by Druid</a>&ensp;·&ensp;
	<a href="/docs/latest/">Docs</a>&ensp;·&ensp;
	<a href="/community/">Community</a>&ensp;·&ensp;
	<a href="/downloads.html">Download</a>&ensp;·&ensp;
	<a href="/faq">FAQ</a>
	</p>
	</div>
	<div class="text-center">
	<a title="Join the user group" href="https://groups.google.com/forum/#!forum/druid-user" target="_blank"><span class="fa fa-comments"></span></a>&ensp;·&ensp;
	<a title="Follow Druid" href="https://twitter.com/druidio" target="_blank"><span class="fab fa-twitter"></span></a>&ensp;·&ensp;
	<a title="GitHub" href="https://github.com/apache/druid" target="_blank"><span class="fab fa-github"></span></a>
	</div>
	<div class="text-center license">
	Copyright © 2020 <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a>.<br>
	Except where otherwise noted, licensed under <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>.<br>
	Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
	</div>
	</div>
	</footer>

	<script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script>
	<script>
	window.dataLayer = window.dataLayer \|\| [];
	function gtag(){dataLayer.push(arguments);}
	gtag('js', new Date());
	gtag('config', 'UA-131010415-1');
	</script>
	<script>
	function trackDownload(type, url) {
	ga('send', 'event', 'download', type, url);
	}
	</script>
	<script src="//code.jquery.com/jquery.min.js"></script>
	<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
	<script src="/assets/js/druid.js"></script>
	<!-- stop page_footer include -->


	<script>
	$(function() {
	$(".toc").load("/docs/0.14.1-incubating/toc.html");

	// There is no way to tell when .gsc-input will be async loaded into the page so just try to set a placeholder until it works
	var tries = 0;
	var timer = setInterval(function() {
	tries++;
	if (tries > 300) clearInterval(timer);
	var searchInput = $('input.gsc-input');
	if (searchInput.length) {
	searchInput.attr('placeholder', 'Search');
	clearInterval(timer);
	}
	}, 100);
	});
	</script>
	</body>
	</html>