| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1" /> |
| <title>Accelerate Apache Spark Applications| Apache Ignite</title> |
| <link rel="stylesheet" href="/js/vendor/hystmodal/hystmodal.min.css" /> |
| <link rel="stylesheet" href="/css/utils.css" /> |
| <link rel="stylesheet" href="/css/site.css" /> |
| <link rel="stylesheet" href="../css/native-persistence.css" /> |
| <link rel="stylesheet" href="../css/compute-apis.css" /> |
| <link rel="stylesheet" href="../css/digital-hub.css" /> |
| <link rel="stylesheet" href="../css/spark.css" /> |
| <link rel="stylesheet" href="/css/media.css" media="only screen and (max-width:1199px)" /> |
| <link rel="icon" type="image/png" href="/img/favicon.png" /> |
| </head> |
| <body> |
| <!-- MOBILE MENU START --> |
| <div class="hystmodal" id="jsMenuModal" aria-hidden="true"> |
| <div class="hystmodal__wrap"> |
| <div class="hystmodal__window mobmenu" role="dialog" aria-modal="true"> |
| <button data-hystclose class="hystmodal__close">Close</button> |
| <div class="mobmenu__wrap"> |
| <a href="/" class="mobmenu__logo"><img src="/img/logo.svg" alt="Логотип" /></a> |
| <div class="mobmenu__menu"> |
| <div class="mobmenu__h1 berlin">Navigation</div> |
| <ul> |
| <li> |
| <a href="/">Get started</a> |
| <ul> |
| <li><a href="/resources.html">Resources</a></li> |
| <li><a href="/faq.html">FAQ</a></li> |
| </ul> |
| </li> |
| <li> |
| <a href="/features/">Features</a> |
| </li> |
| <li> |
| <a href="/community.html">Community</a> |
| <ul> |
| <li><a href="/events.html">Events</a></li> |
| <li><a href="/resources.html">Resources</a></li> |
| </ul> |
| </li> |
| <li> |
| <a href="/use-cases/provenusecases.html">Powered by</a> |
| </li> |
| <li> |
| <a href="https://ignite.apache.org/docs/latest/">Docs</a> |
| </li> |
| </ul> |
| </div> |
| <!-- //mobmenu__menu --> |
| </div> |
| <!-- //mobmenu__wrap --> |
| </div> |
| <!-- //mobmenu --> |
| </div> |
| </div> |
| <!-- MOBILE MENU END --> |
| <header class="hdr hdr__blue jsHdrBase"> |
| <div class="hdr__wrap flexi jsHdrLine"> |
| <button class="hdr__burger" data-menumodal="#jsMenuModal"><img src="/img/menu.svg" alt="" /></button |
| ><a class="hdr__logo" href="/"><img class="hdr__logoimg hdr__logo--white" src="/img/logo-white.svg" alt="" /><img class="hdr__logoimg hdr__logo--black" src="/img/logo.svg" alt="" /></a |
| ><a class="button hdr__button" href="/download.html">Download Ignite 2.11 </a> |
| <nav class="hdrmenu"> |
| <ul class="flexi"> |
| <li><a class="hdrmenu--expanded" href="/" data-panel="getStarted">Get started</a></li> |
| <li><a class="hdrmenu--expanded" href="/features" data-panel="features">Features</a></li> |
| <li><a class="hdrmenu--expanded" href="/community.html" data-panel="community">Community</a></li> |
| <li><a href="/use-cases/provenusecases.html" data-panel="">Powered by</a></li> |
| <li><a href="///ignite.apache.org/docs/latest/" data-panel="">Docs</a></li> |
| </ul> |
| </nav> |
| <div class="dropmenu"> |
| <div class="dropmenu__panel" data-menupanel="getStarted"> |
| <div class="container dropmenu1__grid"> |
| <div class="dropmenu__box dropmenu--left"> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/resources.html"><img src="/img/menu/icon-resources.svg" alt="" /><span>Resources</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/faq.html"><img src="/img/menu/icon-question.svg" alt="" /><span>FAQ</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu1__center"> |
| <div class="dropmenu__box"> |
| <p class="capstext">USE CASES</p> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/in-memory-cache.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>In-Memory Cache</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/in-memory-data-grid.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>In-Memory Data Grid</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/in-memory-database.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>In-Memory Database</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/key-value-store.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>Key-Value Store</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu__box"> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/high-performance-computing.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>High-Performance Computing</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/digital-integration-hub.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>Digital Integration Hub</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/spark-acceleration.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>Spark Acceleration</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/use-cases/hadoop-acceleration.html"><img src="/img/menu/icon-folder.svg" alt="" /><span>Hadoop Acceleration</span></a> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="dropmenu1__right"> |
| <a class="dropmenu__redbutton" href="https://ignite.apache.org/docs/latest/"> <img src="/img/menu/icon-red-rocket.svg" alt="" /><span>Quick Start Guide</span></a> |
| </div> |
| </div> |
| <!-- /.dropmenu1__grid--> |
| </div> |
| <!-- /.dropmenu__panel--> |
| <div class="dropmenu__panel" data-menupanel="features"> |
| <div class="container dropmenu2__grid"> |
| <div class="dropmenu2__col dropmenu__box"> |
| <div class="capstext">Distributed <br />Multi-Tiered Storage</div> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/arch/multi-tier-storage.html"><img src="/img/menu/icon-db.svg" alt="" /><span>Multi-Tier Storage</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/arch/native-persistence.html"><img src="/img/menu/icon-db.svg" alt="" /><span>Native Persistence</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu2__col dropmenu__box"> |
| <div class="capstext">Essentail Developer <br />APIs</div> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/sql.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Distributed SQL</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/distributed-key-value-store.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Key-Value APIs</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/acid-transactions.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>ACID Transactions</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu2__col dropmenu__box"> |
| <div class="capstext">High-Performance <br />Computing APIs</div> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/compute-apis.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Compute APIs</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/servicegrid.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Services</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/machinelearning.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Machine Learning</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu2__col dropmenu__box"> |
| <div class="capstext">Real Streaming APIs</div> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/features/streaming.html"><img src="/img/menu/icon-cube.svg" alt="" /><span>Streaming</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="https://ignite.apache.org/docs/latest/messaging"><img src="/img/menu/icon-cube.svg" alt="" /><span>Messaging</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="https://ignite.apache.org/docs/latest/key-value-api/continuous-queries"><img src="/img/menu/icon-cube.svg" alt="" /><span>Continuous Queries</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu1__right dropmenu2__col"> |
| <a class="dropmenu__redbutton" href="https://ignite.apache.org/docs/latest/"> <img src="/img/menu/icon-red-rocket.svg" alt="" /><span>Quick Start Guide</span></a> |
| </div> |
| </div> |
| <!-- /.dropmenu2__grid--> |
| <div class="panelmorelinkwrap"><a class="panellink" href="/features/">View all</a></div> |
| </div> |
| <!-- /.dropmenu__panel-2--> |
| <div class="dropmenu__panel" data-menupanel="community"> |
| <div class="container dropmenu1__grid dropmenu3__grid"> |
| <div class="dropmenu__box dropmenu--left dropmenu3--left"> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/community.html#story"><img src="/img/menu/icon-clip.svg" alt="" /><span>Apache Ignite Story</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/community.html#community"><img src="/img/menu/icon-comment.svg" alt="" /><span>Meet The Community</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/community.html#contributing"><img src="/img/menu/icon-puzzle.svg" alt="" /><span>Start Contributing</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/community.html#faq"><img src="/img/menu/icon-question.svg" alt="" /><span>Ask A Question</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu1__center"> |
| <div class="dropmenu__box"> |
| <p class="capstext">Events</p> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/events.html#summit"><img src="/img/menu/icon-calendar.svg" alt="" /><span>Apache Ignite Summit</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/events.html#meetups"><img src="/img/menu/icon-calendar.svg" alt="" /><span>Meetups</span></a> |
| </li> |
| </ul> |
| </div> |
| <div class="dropmenu__box"> |
| <ul class="dropmenu__menu"> |
| <li> |
| <a class="dropmenu__iconitem" href="/events.html#upcoming"><img src="/img/menu/icon-calendar.svg" alt="" /><span>Upcoming Events</span></a> |
| </li> |
| <li> |
| <a class="dropmenu__iconitem" href="/events.html#past"><img src="/img/menu/icon-calendar.svg" alt="" /><span>Past Events</span></a> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="dropmenu1__right"> |
| <a class="dropmenu__redbutton" href="https://ignite.apache.org/docs/latest/"> <img src="/img/menu/icon-red-rocket.svg" alt="" /><span>Quick Start Guide</span></a> |
| </div> |
| </div> |
| <!-- /.dropmenu1__grid--> |
| </div> |
| <!-- /.dropmenu__panel--> |
| </div> |
| <!-- /.dropmeu--> |
| </div> |
| </header> |
| <div class="dropmenu__back"></div> |
| <header class="hdrfloat hdr__white jsHdrFloatBase"></header> |
| <section class="innerhero"> |
| <div class="container innerhero__cont"> |
| <div class="innerhero__main"> |
| <h1 class="h1 innerhero__h1"> |
| Accelerate Apache Spark Applications<br /> |
| <span class="with-apache"> With Apache Ignite</span> |
| </h1> |
| <div class="innerhero__descr pt-2 h5"> |
| Minimize data shuffling over the network with the Apache<br /> |
| Ignite implementation of RDD and Dataframe APIs |
| </div> |
| <div class="innerhero__action"><a class="button innerhero__button" href="https://ignite.apache.org/docs/latest/index">Start Coding</a></div> |
| </div> |
| <img class="innerhero__pic innerhero__pic--spark" src="/img/usecases/spark/hero-image.svg" alt="hero-image" /> |
| </div> |
| </section> |
| <!-- /.innerhero--> |
| <section class="spark1"> |
| <div class="container"> |
| <h2 class="h5 spark1__h2">How Does Ignite Accelerate Spark Applications?</h2> |
| <div class="spark1__block"> |
| <div class="spark1__item"> |
| <h3 class="spark1__h3 fz20">Horizontally scalable and shared in-memory layer</h3> |
| <p class="spark1__text">Ignite is designed to store data sets in memory across a cluster of nodes reducing latency of Spark operations that usually need to pull date from disk-based systems.</p> |
| </div> |
| <div class="spark1__item"> |
| <h3 class="spark1__h3 fz20">Minimal data shuffling over the network</h3> |
| <p class="spark1__text">Ignite tries to minimize data shuffling over the network between its store and Spark applications by running certain Spark tasks, produced by RDDs or DataFrames APIs, in-place on Ignite nodes.</p> |
| </div> |
| <div class="spark1__item"> |
| <h3 class="spark1__h3 fz20">Extra performance boost with native Ignite APIs</h3> |
| <p class="spark1__text">Use native Ignite APIs, such as SQL from Spark applications directly and eliminate data shuffling completely between Spark and Ignite.</p> |
| </div> |
| </div> |
| <img class="spark1__image" src="/img/usecases/spark/image.svg" alt="image" /> |
| </div> |
| </section> |
| <!-- /.spark1--> |
| <section class="spark2"> |
| <div class="container"> |
| <h2 class="h5 spark2__h2">Ignite Shared RDDs</h2> |
| <div class="spark2__block"> |
| <div class="spark2__item"> |
| <p class="spark2__text">Apache Ignite provides an implementation of the Spark RDD, which allows any data and state to be shared in memory as RDDs across Spark jobs.</p> |
| <p class="spark2__text pt-2">The Ignite RDD provides a shared, mutable view of the data stored in Ignite caches across different Spark jobs, workers, or applications.</p> |
| </div> |
| <div class="spark2__item"> |
| <p class="spark2__text"> |
| The Ignite RDD is implemented as a view over a distributed Ignite table (aka. cache). It can be deployed with an Ignite node either within the Spark job executing process, on a Spark worker, or in a separate Ignite cluster. |
| </p> |
| <p class="spark2__text pt-2"> |
| This means that depending on the chosen deployment mode, the shared state may either exist only during the lifespan of a Spark application (embedded mode), or it may out-survive the Spark application (standalone mode). |
| </p> |
| </div> |
| </div> |
| <h2 class="h5 spark2__h2">Ignite DataFrames</h2> |
| <div class="spark2__block"> |
| <div class="spark2__item"> |
| <p class="spark2__text">The Apache Spark DataFrame API introduced the concept of a schema to describe the data, allowing Spark to manage the schema and organize the data into a tabular format.</p> |
| <p class="spark2__text pt-2"> |
| To put it simply, a DataFrame is a distributed collection of data organized into named columns. It is conceptually equivalent to a table in a relational database and allows Spark to leverage the Catalyst query optimizer to |
| produce much more efficient query execution plans in comparison to RDDs, which are collections of elements partitioned across the nodes of the cluster. |
| </p> |
| </div> |
| <div class="spark2__item"> |
| <p class="spark2__text">Ignite supports DataFrame APIs allowing Spark to write to and read from Ignite through that interface.</p> |
| <p class="spark2__text pt-1"> |
| Furthermore, Ignite analyses execution plans produced by Spark's Catalyst engine and can execute parts of the plan on Ignite nodes directly, which will reduce data shuffling and consequently make your SparkSQL perform better. |
| </p> |
| </div> |
| </div> |
| </div> |
| </section> |
| <section class="native-bottom container"> |
| <div class="native-bottom__grid"> |
| <article class="nativebotblock"> |
| <h3 class="h4 nativebotblock__title"><img class="nativebotblock__icon" src="/img/features/native-rocket.svg" alt="" /><span>Ready to Start?</span></h3> |
| <p class="nativebotblock__text">Discover our quick start guide and build your first application in 5-10 minutes</p> |
| <a class="nativebotblock__link arrowlink" href="https://ignite.apache.org/docs/latest/" target="_blank">Quick Start Guide</a> |
| </article> |
| <article class="nativebotblock nativebotblock--learn"> |
| <h3 class="h4 nativebotblock__title"><img class="nativebotblock__icon" src="/img/features/native-docs.svg" alt="" /><span>Want to Learn More?</span></h3> |
| <p class="nativebotblock__text">Using Hadoop with Spark? See how Ignite accelerates Hadoop-based deployments</p> |
| <a class="nativebotblock__link arrowlink" href="/use-cases/hadoop-acceleration.html">Apache Hadoop Acceleration Article</a> |
| </article> |
| </div> |
| </section> |
| <footer class="footer"> |
| <div class="container footer__top"> |
| <section class="footer1"> |
| <a class="footer__title" href="/features/">Features</a> |
| <ul class="footer__menu"> |
| <li><a href="/arch/multi-tier-storage.html">Multi-Tier Storage</a></li> |
| <li><a href="/arch/native-persistence.html">Native Persistence</a></li> |
| <li><a href="/features/sql.html">Distributed SQL</a></li> |
| <li><a href="/features/acid-transactions.html">ACID Transactions</a></li> |
| <li><a href="/features/distributed-key-value-store.html">Key Value APIs</a></li> |
| <li><a href="/features/compute-apis.html">Compute APIs</a></li> |
| <li><a href="/features/machinelearning.html">Machine Learning</a></li> |
| <li><a href="/features/servicegrid.html">Services</a></li> |
| <li><a href="/features/streaming.html">Streaming</a></li> |
| <li><a href="https://ignite.apache.org/docs/latest/key-value-api/continuous-queries">Continuous Queries</a></li> |
| <li><a href="https://ignite.apache.org/docs/latest/messaging">Messaging</a></li> |
| </ul> |
| </section> |
| <section class="footer2"> |
| <a class="footer__title" href="/use-cases/">Use-Cases</a> |
| <ul class="footer__menu"> |
| <li><a href="/use-cases/in-memory-cache.html">In-Memory Cache</a></li> |
| <li><a href="/use-cases/in-memory-data-grid.html">In-Memory Data Grid</a></li> |
| <li><a href="/use-cases/in-memory-database.html">In-Memory Database</a></li> |
| <li><a href="/use-cases/high-performance-computing.html">High-Performance Computing</a></li> |
| <li><a href="/use-cases/digital-integration-hub.html">Digital Integration Hub</a></li> |
| <li><a href="/use-cases/key-value-store.html">Key-Value Store</a></li> |
| <li><a href="/use-cases/spark-acceleration.html">Apache Spark Acceleration</a></li> |
| <li><a href="/use-cases/hadoop-acceleration.html">Apache Hadoop Acceleration</a></li> |
| </ul> |
| </section> |
| <section class="footer3"><a class="footer__title" href="/community.html">Community</a><a class="footer__title" href="/resources.html">Resources</a><a class="footer__title" href="/events.html">Events</a></section> |
| <section class="footer4"> |
| <div class="footer4__item">Join Our <a href="mailto:dev@ignite.apache.org">Dev Mailing List</a></div> |
| <div class="footer4__item pt-2">Or write your question</div> |
| <div class="footer4__item pt-1">user@ignite.apache.org</div> |
| <div class="footer4__item pt-5">Follow Us:</div> |
| <div class="footer4__socwrap flexi pt-2"> |
| <a class="footer4__soc" href="https://github.com/apache/ignite" taget="_blank"><img src="/img/icon-github.svg" alt="" /></a |
| ><a class="footer4__soc" href="https://www.linkedin.com/showcase/apache-ignite/" taget="_blank"><img src="/img/icon-linkedin.svg" alt="" /></a |
| ><a class="footer4__soc" href="https://twitter.com/apacheignite" taget="_blank"><img src="/img/icon-twitter.svg" alt="" /></a> |
| </div> |
| <a class="footer4__button button" href="https://ignite.apache.org/docs/latest/index">Start using Apache Ignite </a> |
| </section> |
| </div> |
| <div class="container footer__bot"> |
| <p>© 2015 - 2021 <a href="http://www.apache.org/">The Apache Software Foundation.</a></p> |
| <p class="pt-1x">Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are either registered trademarks or trademarks of The Apache Software Foundation</p> |
| <p class="pt-1x"><a href="http://apache.org/foundation/policies/privacy.html">Privacy Policy</a></p> |
| </div> |
| </footer> |
| <a class="scrollTop" href="#" |
| ><svg class="feather feather-chevron-up" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewbox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> |
| <polyline points="18 15 12 9 6 15"></polyline></svg |
| ></a> |
| <script src="/js/vendor/hystmodal/hystmodal.min.js"></script> |
| <script src="/js/vendor/smoothscroll.js"></script> |
| <script src="/js/main.js"></script> |
| </body> |
| </html> |
