blob: f2c3e1118894c43fc24434588107ec8a73e31963 [file] [log] [blame]
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE html>
<html lang="en">
<head>
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-1382082-1"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-61232409-1');
</script>
<meta charset="UTF-8">
<title>Apache Flume Sink | Ignite Documentation</title>
<link rel="canonical" href="/docs/flume/flume-sink" />
<link rel="stylesheet" href="/assets/css/styles.css?1651672546">
<link rel="stylesheet" href="/assets/css/asciidoc-pygments.css">
<link rel="shortcut icon" href="/favicon.ico">
<meta name='viewport' content='width=device-width, height=device-height, initial-scale=1.0, minimum-scale=1.0'>
<link rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
<script type="text/javascript" src="/assets/js/jquery.swiftype.autocomplete.js?1651672546"></script>
<script type="text/javascript" src="/assets/js/anchor.min.js?1651672546"></script>
</head>
<body>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<header>
<div class="container">
<button type='button' class='menu' title='Docs menu'>
<img src="/assets/images/menu-icon.svg"/>
</button>
<div class='home'>
<a href="/" class='home' title='Apache Ignite home'>
<img src="/assets/images/apache_ignite_logo.svg" alt="Apache Ignite logo" width="103" height="36" >
</a>
</div>
<nav>
</nav>
<form class='search'>
<button class="search-close" type='button'><img src='/assets/images/cancel.svg'></button>
<input type="search" placeholder="Search…" id="search-input">
</form>
<button type='button' class='search-toggle'><img src='/assets/images/search.svg'></button>
<button type='button' class='top-nav-toggle'></button>
<a href="https://github.com/ignite" title='GitHub' class='github' target="_blank">
<img src="/assets/images/github-gray.svg" alt="GitHub logo">
</a>
</div>
</header>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<link rel="stylesheet" href="/assets/css/docs.css">
<section class='page-docs'>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<nav class='left-nav' data-swiftype-index='false'>
<li>
<a href="/docs/extensions/aws/aws" class='' >Amazon S3 IP Finder</a>
</li>
<li>
<a href="/docs/extensions/camel/camel-streamer" class='' >Apache Camel Streamer</a>
</li>
<li>
<a href="/docs/extensions/flink/flink-streamer" class='' >Apache Flink Streamer</a>
</li>
<li>
<a href="/docs/extensions/flume/flume-sink" class='' >Apache Flume Sink</a>
</li>
<li>
<a href="/docs/extensions/azure/azure" class='' >Apache Ignite Azure Module</a>
</li>
<li>
<a href="/docs/extensions/gce/gce" class='' >Apache Ignite GCE Module</a>
</li>
<li>
<a href="/docs/extensions/pub-sub/pub-sub" class='' >Apache Ignite Pub/Sub Module</a>
</li>
<li>
<a href="/docs/extensions/spring/spring-boot" class='' >Apache Ignite and Spring Boot</a>
</li>
<li>
<a href="/docs/extensions/spring/spring-data" class='' >Apache Ignite and Spring Data</a>
</li>
<li>
<a href="/docs/extensions/spring/spring-caching" class='' >Apache Ignite and Spring Cache</a>
</li>
<li>
<a href="/docs/extensions/spring/spring-tx" class='' >Apache Ignite and Spring Transactions</a>
</li>
<li>
<a href="/docs/extensions/kafka/kafka-streamer" class='' >Apache Kafka Streamer</a>
</li>
<li>
<a href="/docs/extensions/storm/storm-streamer" class='' >Apache Storm Streamer</a>
</li>
<li>
<a href="/docs/extensions/cdc/change-data-capture-extensions" class='' >Change Data Capture Extension</a>
</li>
<li>
<a href="/docs/extensions/jms/jms-streamer" class='' >JMS Streamer</a>
</li>
<li>
<a href="/docs/extensions/mqtt/mqtt-streamer" class='' >MQTT Streamer</a>
</li>
<li>
<a href="/docs/extensions/perf-statistics/performance-statistics" class='' >Performance Statistics Extension</a>
</li>
<li>
<a href="/docs/extensions/rocketmq/rocketmq-streamer" class='' >RocketMQ Streamer</a>
</li>
<li>
<a href="/docs/extensions/topology-validator/topology-validator" class='' >Topology Validator</a>
</li>
<li>
<a href="/docs/extensions/twitter/twitter-streamer" class='' >Twitter Streamer</a>
</li>
<li>
<a href="/docs/extensions/zeromq/zeromq-streamer" class='' >ZeroMQ Streamer</a>
</li>
<li>
<a href="/docs/extensions/zookeeper/zookeeper-ip" class='' >ZooKeeper IP Finder</a>
</li>
</nav>
<div class="left-nav__overlay"></div>
<article data-swiftype-index='true'>
<a class='edit-link' href="/_docs/flume/flume-sink.adoc" target="_blank">Edit</a>
<h1>Apache Flume Sink</h1>
<div class="sect1">
<h2 id="overview">Overview</h2>
<div class="sectionbody">
<div class="paragraph">
<p>Apache Flume is a distributed, reliable, and available service for efficiently collecting, aggregating, and moving large
amounts of log data.</p>
</div>
<div class="paragraph">
<p><code>IgniteSink</code> is a Flume sink that extracts events from an associated Flume channel and injects into an Ignite cache.</p>
</div>
<div class="paragraph">
<p><code>IgniteSink</code> and its dependencies have to be included in the agent&#8217;s classpath, as described in the following subsection,
before starting the Flume agent.</p>
</div>
</div>
</div>
<div class="sect1">
<h2 id="setting-up">Setting Up</h2>
<div class="sectionbody">
<div class="olist arabic">
<ol class="arabic">
<li>
<p>Create a transformer by implementing <code>EventTransformer</code> interface.</p>
</li>
<li>
<p>Create <code>ignite</code> directory inside <code>plugins.d</code> directory which is located in <code>${FLUME_HOME}</code>. If the <code>plugins.d</code> directory
is not there, create it.</p>
</li>
<li>
<p>Build it and copy to <code>${FLUME_HOME}/plugins.d/ignite-sink/lib</code>.</p>
</li>
<li>
<p>Copy other Ignite-related jar files from Apache Ignite distribution to <code>${FLUME_HOME}/plugins.d/ignite-sink/libext</code> to
have them as shown below.</p>
<div class="listingblock">
<div class="content">
<pre>plugins.d/
`-- ignite
|-- lib
| `-- ignite-flume-transformer-x.x.x.jar &lt;-- your jar
`-- libext
|-- cache-api-1.0.0.jar
|-- ignite-core-x.x.x.jar
|-- ignite-flume-ext.x.x.x.jar &lt;-- IgniteSink
|-- ignite-spring-x.x.x.jar
|-- spring-aop-4.1.0.RELEASE.jar
|-- spring-beans-4.1.0.RELEASE.jar
|-- spring-context-4.1.0.RELEASE.jar
|-- spring-core-4.1.0.RELEASE.jar
`-- spring-expression-4.1.0.RELEASE.jar</pre>
</div>
</div>
</li>
<li>
<p>In Flume configuration file, specify Ignite configuration XML file&#8217;s location with cache properties
(see <code>flume/src/test/resources/example-ignite.xml</code> for a basic example) with the cache name specified for cache creation.
Also specify the cache name (same as in Ignite configuration file), your `EventTransformer&#8217;s implementation class, and,
optionally, batch size. All properties are shown in the table below (required properties are in bold).</p>
<table class="tableblock frame-all grid-all stripes-even stretch">
<colgroup>
<col style="width: 20%;">
<col style="width: 45%;">
<col style="width: 35%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Property Name</th>
<th class="tableblock halign-left valign-top">Description</th>
<th class="tableblock halign-left valign-top">Default Value</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">channel</p></td>
<td class="tableblock halign-left valign-top"></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">type</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">The component type name. Needs to be <code>org.apache.ignite.stream.flume.IgniteSink</code></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">igniteCfg</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Ignite configuration XML file</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">cacheName</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Cache name. Same as in igniteCfg</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">eventTransformer</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Your implementation of <code>org.apache.ignite.stream.flume.EventTransformer</code></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">batchSize</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Number of events to be written per transaction</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">100</p></td>
</tr>
</tbody>
</table>
</li>
</ol>
</div>
<div class="paragraph">
<p>The sink configuration part of agent named <code>a1</code> can look like this:</p>
</div>
<div class="listingblock">
<div class="content">
<pre>a1.sinks.k1.type = org.apache.ignite.stream.flume.IgniteSink
a1.sinks.k1.igniteCfg = /some-path/ignite.xml
a1.sinks.k1.cacheName = testCache
a1.sinks.k1.eventTransformer = my.company.MyEventTransformer
a1.sinks.k1.batchSize = 100</pre>
</div>
</div>
<div class="paragraph">
<p>After specifying your source and channel (see Flume&#8217;s docs), you are ready to run a Flume agent.</p>
</div>
</div>
</div>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="copyright">
© 2022 The Apache Software Foundation.<br/>
Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are either registered trademarks or trademarks of The Apache Software Foundation.
</div>
</article>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<nav class="right-nav" data-swiftype-index='false'>
<ul class="sectlevel1">
<li><a href="#overview">Overview</a></li>
<li><a href="#setting-up">Setting Up</a></li>
</ul>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<footer>
</footer>
</nav>
</section>
<script type='module' src='/assets/js/code-copy-to-clipboard.js' async></script>
<script>
// inits deep anchors -- needs to be done here because of https://www.bryanbraun.com/anchorjs/#dont-run-it-too-late
anchors.add('.page-docs h1, .page-docs h2, .page-docs h3:not(.discrete), .page-docs h4, .page-docs h5');
anchors.options = {
placement: 'right',
visible: 'always'
};
</script>
</body>
<script type='module' src='/assets/js/index.js?1651672546' async></script>
</html>