| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| <!DOCTYPE html> |
| |
| |
| |
| <html lang="en"> |
| <head> |
| <!-- Global site tag (gtag.js) - Google Analytics --> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-1382082-1"></script> |
| <script> |
| window.dataLayer = window.dataLayer || []; |
| function gtag(){dataLayer.push(arguments);} |
| gtag('js', new Date()); |
| |
| gtag('config', 'UA-61232409-1'); |
| </script> |
| |
| |
| |
| <meta charset="UTF-8"> |
| <title>Apache Flume Sink | Ignite Documentation</title> |
| |
| <link rel="canonical" href="/docs/flume/flume-sink" /> |
| |
| |
| <link rel="stylesheet" href="/assets/css/styles.css?1651672546"> |
| <link rel="stylesheet" href="/assets/css/asciidoc-pygments.css"> |
| <link rel="shortcut icon" href="/favicon.ico"> |
| <meta name='viewport' content='width=device-width, height=device-height, initial-scale=1.0, minimum-scale=1.0'> |
| |
| <link rel="stylesheet" |
| href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"> |
| |
| <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script> |
| <script type="text/javascript" src="/assets/js/jquery.swiftype.autocomplete.js?1651672546"></script> |
| <script type="text/javascript" src="/assets/js/anchor.min.js?1651672546"></script> |
| |
| |
| </head> |
| <body> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| |
| <header> |
| <div class="container"> |
| <button type='button' class='menu' title='Docs menu'> |
| <img src="/assets/images/menu-icon.svg"/> |
| </button> |
| |
| <div class='home'> |
| <a href="/" class='home' title='Apache Ignite home'> |
| <img src="/assets/images/apache_ignite_logo.svg" alt="Apache Ignite logo" width="103" height="36" > |
| </a> |
| </div> |
| |
| <nav> |
| |
| </nav> |
| <form class='search'> |
| <button class="search-close" type='button'><img src='/assets/images/cancel.svg'></button> |
| <input type="search" placeholder="Search…" id="search-input"> |
| </form> |
| <button type='button' class='search-toggle'><img src='/assets/images/search.svg'></button> |
| <button type='button' class='top-nav-toggle'>⋮</button> |
| <a href="https://github.com/ignite" title='GitHub' class='github' target="_blank"> |
| <img src="/assets/images/github-gray.svg" alt="GitHub logo"> |
| </a> |
| </div> |
| </header> |
| |
| |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| <link rel="stylesheet" href="/assets/css/docs.css"> |
| <section class='page-docs'> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| |
| |
| |
| |
| |
| |
| |
| <nav class='left-nav' data-swiftype-index='false'> |
| |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/aws/aws" class='' >Amazon S3 IP Finder</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/camel/camel-streamer" class='' >Apache Camel Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/flink/flink-streamer" class='' >Apache Flink Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/flume/flume-sink" class='' >Apache Flume Sink</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/azure/azure" class='' >Apache Ignite Azure Module</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/gce/gce" class='' >Apache Ignite GCE Module</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/pub-sub/pub-sub" class='' >Apache Ignite Pub/Sub Module</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/spring/spring-boot" class='' >Apache Ignite and Spring Boot</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/spring/spring-data" class='' >Apache Ignite and Spring Data</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/spring/spring-caching" class='' >Apache Ignite and Spring Cache</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/spring/spring-tx" class='' >Apache Ignite and Spring Transactions</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/kafka/kafka-streamer" class='' >Apache Kafka Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/storm/storm-streamer" class='' >Apache Storm Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/cdc/change-data-capture-extensions" class='' >Change Data Capture Extension</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/jms/jms-streamer" class='' >JMS Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/mqtt/mqtt-streamer" class='' >MQTT Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/perf-statistics/performance-statistics" class='' >Performance Statistics Extension</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/rocketmq/rocketmq-streamer" class='' >RocketMQ Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/topology-validator/topology-validator" class='' >Topology Validator</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/twitter/twitter-streamer" class='' >Twitter Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/zeromq/zeromq-streamer" class='' >ZeroMQ Streamer</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/extensions/zookeeper/zookeeper-ip" class='' >ZooKeeper IP Finder</a> |
| |
| </li> |
| |
| </nav> |
| <div class="left-nav__overlay"></div> |
| |
| |
| <article data-swiftype-index='true'> |
| <a class='edit-link' href="/_docs/flume/flume-sink.adoc" target="_blank">Edit</a> |
| |
| <h1>Apache Flume Sink</h1> |
| |
| <div class="sect1"> |
| <h2 id="overview">Overview</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Apache Flume is a distributed, reliable, and available service for efficiently collecting, aggregating, and moving large |
| amounts of log data.</p> |
| </div> |
| <div class="paragraph"> |
| <p><code>IgniteSink</code> is a Flume sink that extracts events from an associated Flume channel and injects into an Ignite cache.</p> |
| </div> |
| <div class="paragraph"> |
| <p><code>IgniteSink</code> and its dependencies have to be included in the agent’s classpath, as described in the following subsection, |
| before starting the Flume agent.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="setting-up">Setting Up</h2> |
| <div class="sectionbody"> |
| <div class="olist arabic"> |
| <ol class="arabic"> |
| <li> |
| <p>Create a transformer by implementing <code>EventTransformer</code> interface.</p> |
| </li> |
| <li> |
| <p>Create <code>ignite</code> directory inside <code>plugins.d</code> directory which is located in <code>${FLUME_HOME}</code>. If the <code>plugins.d</code> directory |
| is not there, create it.</p> |
| </li> |
| <li> |
| <p>Build it and copy to <code>${FLUME_HOME}/plugins.d/ignite-sink/lib</code>.</p> |
| </li> |
| <li> |
| <p>Copy other Ignite-related jar files from Apache Ignite distribution to <code>${FLUME_HOME}/plugins.d/ignite-sink/libext</code> to |
| have them as shown below.</p> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre>plugins.d/ |
| `-- ignite |
| |-- lib |
| | `-- ignite-flume-transformer-x.x.x.jar <-- your jar |
| `-- libext |
| |-- cache-api-1.0.0.jar |
| |-- ignite-core-x.x.x.jar |
| |-- ignite-flume-ext.x.x.x.jar <-- IgniteSink |
| |-- ignite-spring-x.x.x.jar |
| |-- spring-aop-4.1.0.RELEASE.jar |
| |-- spring-beans-4.1.0.RELEASE.jar |
| |-- spring-context-4.1.0.RELEASE.jar |
| |-- spring-core-4.1.0.RELEASE.jar |
| `-- spring-expression-4.1.0.RELEASE.jar</pre> |
| </div> |
| </div> |
| </li> |
| <li> |
| <p>In Flume configuration file, specify Ignite configuration XML file’s location with cache properties |
| (see <code>flume/src/test/resources/example-ignite.xml</code> for a basic example) with the cache name specified for cache creation. |
| Also specify the cache name (same as in Ignite configuration file), your `EventTransformer’s implementation class, and, |
| optionally, batch size. All properties are shown in the table below (required properties are in bold).</p> |
| <table class="tableblock frame-all grid-all stripes-even stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 45%;"> |
| <col style="width: 35%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">Property Name</th> |
| <th class="tableblock halign-left valign-top">Description</th> |
| <th class="tableblock halign-left valign-top">Default Value</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">channel</p></td> |
| <td class="tableblock halign-left valign-top"></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">type</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">The component type name. Needs to be <code>org.apache.ignite.stream.flume.IgniteSink</code></p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">igniteCfg</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Ignite configuration XML file</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">cacheName</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Cache name. Same as in igniteCfg</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">eventTransformer</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Your implementation of <code>org.apache.ignite.stream.flume.EventTransformer</code></p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">batchSize</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Number of events to be written per transaction</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">100</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </li> |
| </ol> |
| </div> |
| <div class="paragraph"> |
| <p>The sink configuration part of agent named <code>a1</code> can look like this:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre>a1.sinks.k1.type = org.apache.ignite.stream.flume.IgniteSink |
| a1.sinks.k1.igniteCfg = /some-path/ignite.xml |
| a1.sinks.k1.cacheName = testCache |
| a1.sinks.k1.eventTransformer = my.company.MyEventTransformer |
| a1.sinks.k1.batchSize = 100</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>After specifying your source and channel (see Flume’s docs), you are ready to run a Flume agent.</p> |
| </div> |
| </div> |
| </div> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| |
| <div class="copyright"> |
| © 2022 The Apache Software Foundation.<br/> |
| Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| |
| </div> |
| |
| </article> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| |
| <nav class="right-nav" data-swiftype-index='false'> |
| <ul class="sectlevel1"> |
| <li><a href="#overview">Overview</a></li> |
| <li><a href="#setting-up">Setting Up</a></li> |
| </ul> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| |
| |
| <footer> |
| </footer> |
| |
| </nav> |
| |
| </section> |
| <script type='module' src='/assets/js/code-copy-to-clipboard.js' async></script> |
| |
| <script> |
| // inits deep anchors -- needs to be done here because of https://www.bryanbraun.com/anchorjs/#dont-run-it-too-late |
| anchors.add('.page-docs h1, .page-docs h2, .page-docs h3:not(.discrete), .page-docs h4, .page-docs h5'); |
| anchors.options = { |
| placement: 'right', |
| visible: 'always' |
| }; |
| </script> |
| </body> |
| <script type='module' src='/assets/js/index.js?1651672546' async></script> |
| </html> |