| <!DOCTYPE html> |
| |
| |
| |
| <html lang="en"> |
| <head> |
| <!-- Global site tag (gtag.js) - Google Analytics --> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-61232409-1"></script> |
| <script> |
| window.dataLayer = window.dataLayer || []; |
| function gtag(){dataLayer.push(arguments);} |
| gtag('js', new Date()); |
| |
| gtag('config', 'UA-61232409-1'); |
| </script> |
| |
| |
| |
| <meta charset="UTF-8"> |
| <meta name="ignite-version" content="2.9.0" /> |
| <title>Installation | Ignite Documentation</title> |
| |
| <link rel="canonical" href="/docs/latest/ignite-for-spark/installation" /> |
| |
| |
| <link rel="stylesheet" href="/assets/css/styles.css?1600372552"> |
| <link rel="stylesheet" href="/assets/css/asciidoc-pygments.css"> |
| <link rel="shortcut icon" href="/favicon.ico"> |
| <meta name='viewport' content='width=device-width, height=device-height, initial-scale=1.0, minimum-scale=1.0'> |
| |
| <script type="text/javascript" src="/assets/js/anchor.min.js?1600372552"></script> |
| |
| <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" /> |
| |
| |
| </head> |
| <body> |
| <header> |
| <div class="container"> |
| <button type='button' class='menu' title='Docs menu'> |
| <img src="/assets/images/menu-icon.svg"/> |
| </button> |
| <div class='home'> |
| <a href="/" class='home' title='Apache Ignite home'> |
| <img src="/assets/images/apache_ignite_logo.svg" alt="Apache Ignite logo" width="103" > |
| </a> |
| </div> |
| |
| <nav> |
| |
| </nav> |
| |
| <select id="version-selector"> |
| <option value="2.9.0">2.9.0</option> |
| </select> |
| |
| <a href="https://github.com/apache/ignite" title='GitHub' class='github' target="_blank"> |
| <img src="/assets/images/github-gray.svg" alt="GitHub logo"> |
| </a> |
| |
| <form class='search'> |
| <button class="search-close" type='button'><img src='/assets/images/cancel.svg'></button> |
| <input type="search" placeholder="Search…" id="search-input"> |
| </form> |
| <button type='button' class='search-toggle'><img src='/assets/images/search.svg'></button> |
| <button type='button' class='top-nav-toggle'>⋮</button> |
| |
| |
| |
| </div> |
| </header> |
| |
| |
| <link rel="stylesheet" href="/assets/css/docs.css"> |
| <section class='page-docs'> |
| |
| |
| |
| |
| <nav class='left-nav' data-swiftype-index='false'> |
| |
| |
| <li> |
| |
| |
| <a href="/docs/latest/preface" class='' >Preface</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Quick Start Guides<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/java" |
| |
| class='' |
| >Java</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/dotnet" |
| |
| class='' |
| >.NET/C#</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/cpp" |
| |
| class='' |
| >C++</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/python" |
| |
| class='' |
| >Python</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/nodejs" |
| |
| class='' |
| >Node.JS</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/sql" |
| |
| class='' |
| >SQL</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/php" |
| |
| class='' |
| >PHP</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/quick-start/restapi" |
| |
| class='' |
| >REST API</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="/installation" class='group-toggle collapsed parent'>Installation<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/installation/installing-using-zip" |
| |
| class='' |
| >Installing Using ZIP Archive</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/installation/installing-using-docker" |
| |
| class='' |
| >Installing Using Docker</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/installation/deb-rpm" |
| |
| class='' |
| >Installing DEB or RPM package</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Kubernetes<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//installation/kubernetes/amazon-eks-deployment" class=''>Amazon EKS</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//installation/kubernetes/azure-deployment" class=''>Azure Kubernetes Service</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//installation/kubernetes/gke-deployment" class=''>Google Kubernetes Engine</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Setting Up<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/setup" |
| |
| class='' |
| >Setting Up Ignite for Java</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/setup-dotnet" |
| |
| class='' |
| >Setting Up Ignite for .NET/C#</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/understanding-configuration" class='' >Understanding Configuration</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/logging" class='' >Configuring Logging</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/starting-nodes" class='' >Starting and Stopping Nodes</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Clustering<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/clustering" |
| |
| class='' |
| >Overview</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/tcp-ip-discovery" |
| |
| class='' |
| >TCP/IP Discovery</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/zookeeper-discovery" |
| |
| class='' |
| >ZooKeeper Discovery</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/discovery-in-the-cloud" |
| |
| class='' |
| >Discovery in the Cloud</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/network-configuration" |
| |
| class='' |
| >Network Configuration</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/connect-client-nodes" |
| |
| class='' |
| >Connecting Client Nodes</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/clustering/running-client-nodes-behind-nat" |
| |
| class='' |
| >Running Client Nodes Behind NAT</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Data Modeling<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-modeling/data-modeling" |
| |
| class='' |
| >Introduction</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-modeling/data-partitioning" |
| |
| class='' |
| >Data Partitioning</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-modeling/affinity-collocation" |
| |
| class='' |
| >Affinity Colocation</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Configuring Memory<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/memory-architecture" |
| |
| class='' |
| >Memory Architecture</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/memory-configuration/data-regions" |
| |
| class='' |
| >Configuring Data Regions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/memory-configuration/eviction-policies" |
| |
| class='' |
| >Eviction Policies</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Configuring Caches<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/configuration-overview" |
| |
| class='' |
| >Cache Configuration</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/configuring-backups" |
| |
| class='' |
| >Configuring Partition Backups</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/atomicity-modes" |
| |
| class='' |
| >Atomicity Modes</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/expiry-policies" |
| |
| class='' |
| >Expiry Policy</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/on-heap-caching" |
| |
| class='' |
| >On-Heap Caching</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/configuring-caches/cache-groups" |
| |
| class='' |
| >Cache Groups</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Persistence<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/native-persistence" |
| |
| class='' |
| >Ignite Persistence</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/external-storage" |
| |
| class='' |
| >External Storage</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/swap" |
| |
| class='' |
| >Swapping</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/custom-cache-store" |
| |
| class='' |
| >Implementing Custom Cache Store</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/snapshots" |
| |
| class='' |
| >Cluster Snapshots</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/disk-compression" |
| |
| class='' |
| >Disk Compression</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/persistence/persistence-tuning" |
| |
| class='' |
| >Tuning Persistence</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/baseline-topology" class='' >Baseline Topology</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/cluster-states" class='' >Cluster States</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/data-rebalancing" class='' >Data Rebalancing</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/partition-loss-policy" class='' >Partition Loss Policy</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/deploying-user-code" class='' >Deploying User Code</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/peer-class-loading" class='' >Peer Class Loading</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/data-streaming" class='' >Data Streaming</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Using Key-Value Cache API<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/key-value-api/basic-cache-operations" |
| |
| class='' |
| >Basic Cache Operations</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/key-value-api/binary-objects" |
| |
| class='' |
| >Working with Binary Objects</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/key-value-api/using-scan-queries" |
| |
| class='' |
| >Using Scan Queries</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/read-repair" |
| |
| class='' |
| >Read Repair</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/key-value-api/continuous-queries" class='' >Using Continuous Queries</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/key-value-api/transactions" class='' >Performing Transactions</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Working with SQL<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/sql-introduction" |
| |
| class='' |
| >Introduction</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/schemas" |
| |
| class='' |
| >Understanding Schemas</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/indexes" |
| |
| class='' |
| >Defining Indexes</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/sql-api" |
| |
| class='' |
| >Using SQL API</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/distributed-joins" |
| |
| class='' |
| >Distributed Joins</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/sql-transactions" |
| |
| class='' |
| >SQL Transactions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/custom-sql-func" |
| |
| class='' |
| >Custom SQL Functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/JDBC/jdbc-driver" |
| |
| class='' |
| >JDBC Driver</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/JDBC/jdbc-client-driver" |
| |
| class='' |
| >JDBC Client Driver</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/transactions/mvcc" |
| |
| class='' |
| >Multiversion Concurrency Control</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Distributed Computing<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/distributed-computing" |
| |
| class='' |
| >Distributed Computing API</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/cluster-groups" |
| |
| class='' |
| >Cluster Groups</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/executor-service" |
| |
| class='' |
| >Executor Service</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/map-reduce" |
| |
| class='' |
| >MapReduce API</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/load-balancing" |
| |
| class='' |
| >Load Balancing</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/fault-tolerance" |
| |
| class='' |
| >Fault Tolerance</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/distributed-computing/job-scheduling" |
| |
| class='' |
| >Job Scheduling</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/collocated-computations" class='' >Colocating Computations with Data</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Working with Events<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/events/listening-to-events" |
| |
| class='' |
| >Enabling and Listenting to Events</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/events/events" |
| |
| class='' |
| >Events</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/near-cache" class='' >Near Caches</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/platform-cache" class='' >.NET Platform Cache</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/services/services" class='' >Services</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Distributed Data Structures<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-structures/queue-and-set" |
| |
| class='' |
| >Queue and Set</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-structures/atomic-types" |
| |
| class='' |
| >Atomic Types</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-structures/countdownlatch" |
| |
| class='' |
| >CountDownLatch</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-structures/atomic-sequence" |
| |
| class='' |
| >Atomic Sequence</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/data-structures/semaphore" |
| |
| class='' |
| >Semaphore</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Machine Learning<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/machine-learning" |
| |
| class='' |
| >Machine Learning</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/partition-based-dataset" |
| |
| class='' |
| >Partition Based Dataset</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/updating-trained-models" |
| |
| class='' |
| >Updating Trained Models</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Binary Classification<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/introduction" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/linear-svm" class=''>Linear SVM (Support Vector Machine)</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/decision-trees" class=''>Decision Trees</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/multilayer-perceptron" class=''>Multilayer Perceptron</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/logistic-regression" class=''>Logistic Regression</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/knn-classification" class=''>k-NN Classification</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/ann" class=''>ANN (Approximate Nearest Neighbor)</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/binary-classification/naive-bayes" class=''>Naive Bayes</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Regression<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/regression/introduction" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/regression/linear-regression" class=''>Linear Regression</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/regression/decision-trees-regression" class=''>Decision Trees Regression</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/regression/knn-regression" class=''>k-NN Regression</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Clustering<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/clustering/introduction" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/clustering/k-means-clustering" class=''>K-Means Clustering</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/clustering/gaussian-mixture" class=''>Gaussian mixture (GMM)</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/preprocessing" |
| |
| class='' |
| >Preprocessing</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Model Selection<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/model-selection/introduction" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/model-selection/evaluator" class=''>Evaluator</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/model-selection/split-the-dataset-on-test-and-train-datasets" class=''>Split the dataset on test and train datasets</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/model-selection/hyper-parameter-tuning" class=''>Hyper-parameter tuning</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/model-selection/pipeline-api" class=''>Pipeline API</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/multiclass-classification" |
| |
| class='' |
| >Multiclass Classification</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Ensemble Methods<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/ensemble-methods/introduction" class=''></a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/ensemble-methods/stacking" class=''>Stacking</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/ensemble-methods/baggin" class=''>Bagging</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/ensemble-methods/random-forest" class=''>Random Forest</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/ensemble-methods/gradient-boosting" class=''>Gradient Boosting</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/machine-learning/recommendation-systems" |
| |
| class='' |
| >Recommendation Systems</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Importing Model<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/importing-model/introduction" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/importing-model/model-import-from-gxboost" class=''>Import Model from XGBoost</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//machine-learning/importing-model/model-import-from-apache-spark" class=''>Import Model from Apache Spark</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Monitoring<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/monitoring-metrics/intro" |
| |
| class='' |
| >Introduction</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/monitoring-metrics/cluster-id" |
| |
| class='' |
| >Cluster ID and Tag</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Metrics<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//monitoring-metrics/configuring-metrics" class=''>Configuring Metrics</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//monitoring-metrics/metrics" class=''>JMX Metrics</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>New Metrics System<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//monitoring-metrics/new-metrics-system" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//monitoring-metrics/new-metrics" class=''>Metrics</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/monitoring-metrics/system-views" |
| |
| class='' |
| >System Views</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/monitoring-metrics/tracing" |
| |
| class='' |
| >Tracing</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="/security" class='group-toggle collapsed '>Security<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/security/authentication" |
| |
| class='' |
| >Authentication</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/security/ssl-tls" |
| |
| class='' |
| >SSL/TLS</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Transparent Data Encryption<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//security/tde" class=''>Introduction</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//security/master-key-rotation" class=''>Master key rotation</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Thin Clients<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/getting-started-with-thin-clients" |
| |
| class='' |
| >Thin Clients Overview</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/java-thin-client" |
| |
| class='' |
| >Java Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/dotnet-thin-client" |
| |
| class='' |
| >.NET Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/cpp-thin-client" |
| |
| class='' |
| >C++ Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/python-thin-client" |
| |
| class='' |
| >Python Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/php-thin-client" |
| |
| class='' |
| >PHP Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/thin-clients/nodejs-thin-client" |
| |
| class='' |
| >Node.js Thin Client</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| |
| |
| <button |
| type='button' |
| class='collapsed '>Binary Client Protocol<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class="sub_pages collapsed"> |
| |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/binary-client-protocol" class=''>Binary Client Protocol</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/data-format" class=''>Data Format</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/key-value-queries" class=''>Key-Value Queries</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/sql-and-scan-queries" class=''>SQL and Scan Queries</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/binary-type-metadata" class=''>Binary Types Metadata</a></li> |
| |
| |
| |
| <li><a href="/docs/latest//binary-client-protocol/cache-configuration" class=''>Cache Configuration</a></li> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>ODBC Driver<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/odbc-driver" |
| |
| class='' |
| >ODBC Driver</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/connection-string-dsn" |
| |
| class='' |
| >Connection String and DSN</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/querying-modifying-data" |
| |
| class='' |
| >Querying and Modifying Data</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/specification" |
| |
| class='' |
| >Specification</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/data-types" |
| |
| class='' |
| >Data Types</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/SQL/ODBC/error-codes" |
| |
| class='' |
| >Error Codes</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/restapi" class='' >REST API</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/control-script" class='' >Control Script</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/plugins" class='' >Plugins</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| <a href="/docs/latest/sqlline" class='' >SQLLine</a> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="" class='group-toggle collapsed '>Ignite for Spark<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/overview" |
| |
| class='' |
| >Overview</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/ignitecontext-and-rdd" |
| |
| class='' |
| >IgniteContext and IgniteRDD</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/ignite-dataframe" |
| |
| class='' |
| >Ignite DataFrame</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/installation" |
| |
| class='' |
| >Installation</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/spark-shell" |
| |
| class='' |
| >Test Ignite with Spark-shell</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/ignite-for-spark/troubleshooting" |
| |
| class='' |
| >Troubleshooting</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| <li> |
| |
| |
| |
| |
| |
| |
| <button type='button' data-guide-url="/sql-reference/sql-reference-overview" class='group-toggle collapsed '>SQL Reference<img class="state-indicator" src="/assets/images/left-nav-arrow.svg"></button> |
| <nav class='nav-group collapsed'> |
| |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/sql-conformance" |
| |
| class='' |
| >SQL Conformance</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/ddl" |
| |
| class='' |
| >Data Definition Language (DDL)</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/dml" |
| |
| class='' |
| >Data Manipulation Language (DML)</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/transactions" |
| |
| class='' |
| >Transactions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/operational-commands" |
| |
| class='' |
| >Operational Commands</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/aggregate-functions" |
| |
| class='' |
| >Aggregate functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/numeric-functions" |
| |
| class='' |
| >Numeric Functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/string-functions" |
| |
| class='' |
| >String Functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/date-time-functions" |
| |
| class='' |
| >Data and Time Functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/system-functions" |
| |
| class='' |
| >System Functions</a> |
| |
| </li> |
| |
| |
| |
| |
| |
| <li> |
| |
| <a href="/docs/latest/sql-reference/data-types" |
| |
| class='' |
| >Data Types</a> |
| |
| </li> |
| |
| |
| |
| </nav> |
| |
| </li> |
| |
| </nav> |
| <div class="left-nav__overlay"></div> |
| |
| |
| <article data-swiftype-index='true'> |
| <a class='edit-link' href="https://github.com/apache/ignite/tree/IGNITE-7595/docs/_docs/ignite-for-spark/installation.adoc" target="_blank">Edit</a> |
| |
| <h1>Installation</h1> |
| |
| <div class="sect1"> |
| <h2 id="shared-deployment">Shared Deployment</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Shared deployment implies that Apache Ignite nodes are running independently from Apache Spark applications and store state even after Apache Spark jobs die. Similarly to Apache Spark, there are three ways to deploy Apache Ignite to the cluster.</p> |
| </div> |
| <div class="sect2"> |
| <h3 id="standalone-deployment">Standalone Deployment</h3> |
| <div class="paragraph"> |
| <p>In the Standalone deployment mode, Ignite nodes should be deployed together with Spark Worker nodes. Instruction on Ignite installation can be found <a href="/docs/latest/installation">here</a>. After you install Ignite on all worker nodes, start a node on each Spark worker with your config using <code>ignite.sh</code> script.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="adding-ignite-libraries-to-spark-classpath-by-default">Adding Ignite libraries to Spark classpath by default</h3> |
| <div class="paragraph"> |
| <p>Spark application deployment model allows dynamic jar distribution during application start. This model, however, has some drawbacks:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Spark dynamic class loader does not implement <code>getResource</code> methods, so you will not be able to access resources located in jar files.</p> |
| </li> |
| <li> |
| <p>Java logger uses application class loader (not the context class loader) to load log handlers which results in <code>ClassNotFoundException</code> when using Java logging in Ignite.</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>There is a way to alter the default Spark classpath for each launched application (this should be done on each machine of the Spark cluster, including master, worker and driver nodes).</p> |
| </div> |
| <div class="olist arabic"> |
| <ol class="arabic"> |
| <li> |
| <p>Locate the <code>$SPARK_HOME/conf/spark-env.sh</code> file. If this file does not exist, create it from template using <code>$SPARK_HOME/conf/spark-env.sh.template</code></p> |
| </li> |
| <li> |
| <p>Add the following lines to the end of the <code>spark-env.sh</code> file (uncomment the line setting <code>IGNITE_HOME</code> in case if you do not have it globally set):</p> |
| </li> |
| </ol> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="shell"><span class="c"># Optionally set IGNITE_HOME here.</span> |
| <span class="c"># IGNITE_HOME=/path/to/ignite</span> |
| |
| <span class="nv">IGNITE_LIBS</span><span class="o">=</span><span class="s2">"</span><span class="k">${</span><span class="nv">IGNITE_HOME</span><span class="k">}</span><span class="s2">/libs/*"</span> |
| |
| <span class="k">for </span>file <span class="k">in</span> <span class="k">${</span><span class="nv">IGNITE_HOME</span><span class="k">}</span>/libs/<span class="k">*</span> |
| <span class="k">do |
| if</span> <span class="o">[</span> <span class="nt">-d</span> <span class="k">${</span><span class="nv">file</span><span class="k">}</span> <span class="o">]</span> <span class="o">&&</span> <span class="o">[</span> <span class="s2">"</span><span class="k">${</span><span class="nv">file</span><span class="k">}</span><span class="s2">"</span> <span class="o">!=</span> <span class="s2">"</span><span class="k">${</span><span class="nv">IGNITE_HOME</span><span class="k">}</span><span class="s2">"</span>/libs/optional <span class="o">]</span><span class="p">;</span> <span class="k">then |
| </span><span class="nv">IGNITE_LIBS</span><span class="o">=</span><span class="k">${</span><span class="nv">IGNITE_LIBS</span><span class="k">}</span>:<span class="k">${</span><span class="nv">file</span><span class="k">}</span>/<span class="k">*</span> |
| <span class="k">fi |
| done |
| |
| </span><span class="nb">export </span><span class="nv">SPARK_CLASSPATH</span><span class="o">=</span><span class="nv">$IGNITE_LIBS</span></code></pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>Copy any folders required from the <code>$IGNITE_HOME/libs/optional</code> folder, such as <code>ignite-log4j</code>, to the <code>$IGNITE_HOME/libs</code> folder.</p> |
| </div> |
| <div class="paragraph"> |
| <p>You can verify that the Spark classpath is changed by running <code>bin/spark-shell</code> and typing a simple import statement:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="shell">scala> import org.apache.ignite.configuration._ |
| import org.apache.ignite.configuration._</code></pre> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="embedded-deployment">Embedded Deployment</h2> |
| <div class="sectionbody"> |
| <div class="admonitionblock caution"> |
| <table> |
| <tr> |
| <td class="icon"> |
| <div class="title">Caution</div> |
| </td> |
| <td class="content"> |
| <h3 id="embedded-mode-deprecation" class="discrete">Embedded Mode Deprecation</h3> |
| <div class="paragraph"> |
| <p>Embedded mode implies starting Ignite server nodes within Spark executors which can cause unexpected rebalancing or even data loss. Therefore this mode is currently deprecated and will be eventually discontinued. Consider starting a separate Ignite cluster and using standalone mode to avoid data consistency and performance issues.</p> |
| </div> |
| </td> |
| </tr> |
| </table> |
| </div> |
| <div class="paragraph"> |
| <p>Embedded deployment means that Apache Ignite nodes are started inside the Apache Spark job processes and are stopped when the job dies. There is no need for additional deployment steps in this case. Apache Ignite code will be distributed to worker machines using the Apache Spark deployment mechanism and nodes will be started on all workers as part of the <code>IgniteContext</code> initialization.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="maven">Maven</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Ignite’s Spark artifact is <a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.ignite%22" target="_blank" rel="noopener">hosted in Maven Central</a>. Depending on a Scala version you use, include the artifact using one of the dependencies shown below.</p> |
| </div> |
| <div class="listingblock"> |
| <div class="title">Scala 2.11</div> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="scala"><span class="o"><</span><span class="n">dependency</span><span class="o">></span> |
| <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="nv">org</span><span class="o">.</span><span class="py">apache</span><span class="o">.</span><span class="py">ignite</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> |
| <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">ignite</span><span class="o">-</span><span class="n">spark</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> |
| <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="n">$</span><span class="o">{</span><span class="nv">ignite</span><span class="o">.</span><span class="py">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span> |
| <span class="o"></</span><span class="n">dependency</span><span class="o">></span></code></pre> |
| </div> |
| </div> |
| <div class="listingblock"> |
| <div class="title">Scala 2.10</div> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="scala"><span class="o"><</span><span class="n">dependency</span><span class="o">></span> |
| <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="nv">org</span><span class="o">.</span><span class="py">apache</span><span class="o">.</span><span class="py">ignite</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> |
| <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">ignite</span><span class="o">-</span><span class="n">spark_2</span><span class="o">.</span><span class="mi">10</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> |
| <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="n">$</span><span class="o">{</span><span class="nv">ignite</span><span class="o">.</span><span class="py">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span> |
| <span class="o"></</span><span class="n">dependency</span><span class="o">></span></code></pre> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="sbt">SBT</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>If SBT is used as a build tool for a Scala application, then Ignite’s Spark artifact can be added into <code>build.sbt</code> with one of the commands below:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="title">Scala 2.11</div> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="scala"><span class="n">libraryDependencies</span> <span class="o">+=</span> <span class="s">"org.apache.ignite"</span> <span class="o">%</span> <span class="s">"ignite-spark"</span> <span class="o">%</span> <span class="s">"ignite.version"</span></code></pre> |
| </div> |
| </div> |
| <div class="listingblock"> |
| <div class="title">Scala 2.10</div> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="scala"><span class="n">libraryDependencies</span> <span class="o">+=</span> <span class="s">"org.apache.ignite"</span> <span class="o">%</span> <span class="s">"ignite-spark_2.10"</span> <span class="o">%</span> <span class="s">"ignite.version"</span></code></pre> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="classpath-configuration">Classpath Configuration</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>When IgniteRDD or Ignite Data Frames APIs are used, make sure that Spark executors and drivers have all the required Ignite jars available in their classpath. Spark provides several ways to modify the classpath of both the driver or the executor process.</p> |
| </div> |
| <div class="sect2"> |
| <h3 id="parameters-configuration">Parameters Configuration</h3> |
| <div class="paragraph"> |
| <p>Ignite jars can be added to Spark using configuration parameters such as |
| <code>spark.driver.extraClassPath</code> and <code>spark.executor.extraClassPath</code>. Refer to the <a href="https://spark.apache.org/docs/latest/configuration.html#runtime-environment">Spark official documentation</a> for all available options.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The following shows how to fill in <code>spark.driver.extraClassPath</code> parameters:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="shell">spark.executor.extraClassPath /opt/ignite/libs/<span class="k">*</span>:/opt/ignite/libs/optional/ignite-spark/<span class="k">*</span>:/opt/ignite/libs/optional/ignite-log4j/<span class="k">*</span>:/opt/ignite/libs/optional/ignite-yarn/<span class="k">*</span>:/opt/ignite/libs/ignite-spring/<span class="k">*</span></code></pre> |
| </div> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="source-code-configuration">Source Code Configuration</h3> |
| <div class="paragraph"> |
| <p>Spark provides APIs to set up extra libraries from the application code. You can provide Ignite jars in the following way:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="rouge highlight"><code data-lang="scala"><span class="k">private</span> <span class="k">val</span> <span class="nv">MAVEN_HOME</span> <span class="k">=</span> <span class="s">"/home/user/.m2/repository"</span> |
| |
| <span class="k">val</span> <span class="nv">spark</span> <span class="k">=</span> <span class="nv">SparkSession</span><span class="o">.</span><span class="py">builder</span><span class="o">()</span> |
| <span class="o">.</span><span class="py">appName</span><span class="o">(</span><span class="s">"Spark Ignite data sources example"</span><span class="o">)</span> |
| <span class="o">.</span><span class="py">master</span><span class="o">(</span><span class="s">"spark://172.17.0.2:7077"</span><span class="o">)</span> |
| <span class="o">.</span><span class="py">getOrCreate</span><span class="o">()</span> |
| |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/apache/ignite/ignite-core/2.4.0/ignite-core-2.4.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/apache/ignite/ignite-spring/2.4.0/ignite-spring-2.4.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/apache/ignite/ignite-log4j/2.4.0/ignite-log4j-2.4.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/apache/ignite/ignite-spark/2.4.0/ignite-spark-2.4.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/apache/ignite/ignite-indexing/2.4.0/ignite-indexing-2.4.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/springframework/spring-beans/4.3.7.RELEASE/spring-beans-4.3.7.RELEASE.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/springframework/spring-core/4.3.7.RELEASE/spring-core-4.3.7.RELEASE.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/springframework/spring-context/4.3.7.RELEASE/spring-context-4.3.7.RELEASE.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/org/springframework/spring-expression/4.3.7.RELEASE/spring-expression-4.3.7.RELEASE.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/javax/cache/cache-api/1.0.0/cache-api-1.0.0.jar"</span><span class="o">)</span> |
| <span class="nv">spark</span><span class="o">.</span><span class="py">sparkContext</span><span class="o">.</span><span class="py">addJar</span><span class="o">(</span><span class="nc">MAVEN_HOME</span> <span class="o">+</span> <span class="s">"/com/h2database/h2/1.4.195/h2-1.4.195.jar"</span><span class="o">)</span></code></pre> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="copyright"> |
| © 2020 The Apache Software Foundation.<br/> |
| Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| |
| </div> |
| |
| </article> |
| <nav class="right-nav" data-swiftype-index='false'> |
| <ul class="sectlevel1"> |
| <li><a href="#shared-deployment">Shared Deployment</a> |
| <ul class="sectlevel2"> |
| <li><a href="#standalone-deployment">Standalone Deployment</a></li> |
| <li><a href="#adding-ignite-libraries-to-spark-classpath-by-default">Adding Ignite libraries to Spark classpath by default</a></li> |
| </ul> |
| </li> |
| <li><a href="#embedded-deployment">Embedded Deployment</a></li> |
| <li><a href="#maven">Maven</a></li> |
| <li><a href="#sbt">SBT</a></li> |
| <li><a href="#classpath-configuration">Classpath Configuration</a> |
| <ul class="sectlevel2"> |
| <li><a href="#parameters-configuration">Parameters Configuration</a></li> |
| <li><a href="#source-code-configuration">Source Code Configuration</a></li> |
| </ul> |
| </li> |
| </ul> |
| |
| |
| <footer> |
| </footer> |
| |
| </nav> |
| |
| </section> |
| <script type='module' src='/assets/js/code-copy-to-clipboard.js' async crossorigin></script> |
| |
| <script> |
| // inits deep anchors -- needs to be done here because of https://www.bryanbraun.com/anchorjs/#dont-run-it-too-late |
| anchors.add('.page-docs h1, .page-docs h2, .page-docs h3:not(.discrete), .page-docs h4, .page-docs h5'); |
| anchors.options = { |
| placement: 'right', |
| visible: 'always' |
| }; |
| </script> |
| <!-- load google fonts async --> |
| <script type="text/javascript"> |
| WebFontConfig = { |
| google: { families: [ 'Open+Sans:300,400,600,700&display=swap' ] } |
| }; |
| (function() { |
| var wf = document.createElement('script'); |
| wf.src = 'https://ajax.googleapis.com/ajax/libs/webfont/1/webfont.js'; |
| wf.type = 'text/javascript'; |
| wf.async = 'true'; |
| var s = document.getElementsByTagName('script')[0]; |
| s.parentNode.insertBefore(wf, s); |
| })(); </script> |
| <script src="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js"></script> |
| <script> |
| docsearch({ |
| // Your apiKey and indexName will be given to you once |
| // we create your config |
| apiKey: '3eee686c0ebe39eff3baeb18c56fa5f8', |
| indexName: 'apache_ignite', |
| // Replace inputSelector with a CSS selector |
| // matching your search input |
| inputSelector: '#search-input', |
| // algoliaOptions: { 'facetFilters': ["version:$VERSION"] }, |
| |
| // Set debug to true to inspect the dropdown |
| debug: false, |
| }); |
| </script> |
| <script type='module' src='/assets/js/index.js?1600372552' async crossorigin></script> |
| <script type='module' src='/assets/js/versioning.js?1600372552' async crossorigin></script> |
| </body> |
| |
| </html> |
