| |
| <!DOCTYPE html> |
| <!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]--> |
| <!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]--> |
| <!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Submitting Applications - Spark 3.5.0 Documentation</title> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="css/bootstrap.min.css"> |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet"> |
| <link href="css/custom.css" rel="stylesheet"> |
| <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script> |
| |
| <link rel="stylesheet" href="css/pygments-default.css"> |
| <link rel="stylesheet" href="css/docsearch.min.css" /> |
| <link rel="stylesheet" href="css/docsearch.css"> |
| |
| <!-- Matomo --> |
| <script type="text/javascript"> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '40']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| </head> |
| <body class="global"> |
| <!--[if lt IE 7]> |
| <p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p> |
| <![endif]--> |
| |
| <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html --> |
| |
| <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar"> |
| <div class="navbar-brand"><a href="index.html"> |
| <img src="img/spark-logo-rev.svg" width="141" height="72"/></a><span class="version">3.5.0</span> |
| </div> |
| <button class="navbar-toggler" type="button" data-toggle="collapse" |
| data-target="#navbarCollapse" aria-controls="navbarCollapse" |
| aria-expanded="false" aria-label="Toggle navigation"> |
| <span class="navbar-toggler-icon"></span> |
| </button> |
| <div class="collapse navbar-collapse" id="navbarCollapse"> |
| <ul class="navbar-nav me-auto"> |
| <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a> |
| <div class="dropdown-menu" aria-labelledby="navbarQuickStart"> |
| <a class="dropdown-item" href="quick-start.html">Quick Start</a> |
| <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a> |
| <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a> |
| <a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a> |
| <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a> |
| <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a> |
| <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a> |
| <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a> |
| <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a> |
| <div class="dropdown-menu" aria-labelledby="navbarAPIDocs"> |
| <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a> |
| <a class="dropdown-item" href="api/java/index.html">Java</a> |
| <a class="dropdown-item" href="api/python/index.html">Python</a> |
| <a class="dropdown-item" href="api/R/index.html">R</a> |
| <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a> |
| <div class="dropdown-menu" aria-labelledby="navbarDeploying"> |
| <a class="dropdown-item" href="cluster-overview.html">Overview</a> |
| <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a> |
| <div class="dropdown-divider"></div> |
| <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a> |
| <a class="dropdown-item" href="running-on-mesos.html">Mesos</a> |
| <a class="dropdown-item" href="running-on-yarn.html">YARN</a> |
| <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a> |
| <div class="dropdown-menu" aria-labelledby="navbarMore"> |
| <a class="dropdown-item" href="configuration.html">Configuration</a> |
| <a class="dropdown-item" href="monitoring.html">Monitoring</a> |
| <a class="dropdown-item" href="tuning.html">Tuning Guide</a> |
| <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a> |
| <a class="dropdown-item" href="security.html">Security</a> |
| <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a> |
| <a class="dropdown-item" href="migration-guide.html">Migration Guide</a> |
| <div class="dropdown-divider"></div> |
| <a class="dropdown-item" href="building-spark.html">Building Spark</a> |
| <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a> |
| <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a> |
| </div> |
| </li> |
| |
| <li class="nav-item"> |
| <input type="text" id="docsearch-input" placeholder="Search the docs…"> |
| </li> |
| </ul> |
| <!--<span class="navbar-text navbar-right"><span class="version-text">v3.5.0</span></span>--> |
| </div> |
| </nav> |
| |
| |
| |
| <div class="container"> |
| |
| |
| <div class="content mr-3" id="content"> |
| |
| |
| <h1 class="title">Submitting Applications</h1> |
| |
| |
| <p>The <code class="language-plaintext highlighter-rouge">spark-submit</code> script in Spark’s <code class="language-plaintext highlighter-rouge">bin</code> directory is used to launch applications on a cluster. |
| It can use all of Spark’s supported <a href="cluster-overview.html#cluster-manager-types">cluster managers</a> |
| through a uniform interface so you don’t have to configure your application especially for each one.</p> |
| |
| <h1 id="bundling-your-applications-dependencies">Bundling Your Application’s Dependencies</h1> |
| <p>If your code depends on other projects, you will need to package them alongside |
| your application in order to distribute the code to a Spark cluster. To do this, |
| create an assembly jar (or “uber” jar) containing your code and its dependencies. Both |
| <a href="https://github.com/sbt/sbt-assembly">sbt</a> and |
| <a href="http://maven.apache.org/plugins/maven-shade-plugin/">Maven</a> |
| have assembly plugins. When creating assembly jars, list Spark and Hadoop |
| as <code class="language-plaintext highlighter-rouge">provided</code> dependencies; these need not be bundled since they are provided by |
| the cluster manager at runtime. Once you have an assembled jar you can call the <code class="language-plaintext highlighter-rouge">bin/spark-submit</code> |
| script as shown here while passing your jar.</p> |
| |
| <p>For Python, you can use the <code class="language-plaintext highlighter-rouge">--py-files</code> argument of <code class="language-plaintext highlighter-rouge">spark-submit</code> to add <code class="language-plaintext highlighter-rouge">.py</code>, <code class="language-plaintext highlighter-rouge">.zip</code> or <code class="language-plaintext highlighter-rouge">.egg</code> |
| files to be distributed with your application. If you depend on multiple Python files we recommend |
| packaging them into a <code class="language-plaintext highlighter-rouge">.zip</code> or <code class="language-plaintext highlighter-rouge">.egg</code>. For third-party Python dependencies, |
| see <a href="api/python/user_guide/python_packaging.html">Python Package Management</a>.</p> |
| |
| <h1 id="launching-applications-with-spark-submit">Launching Applications with spark-submit</h1> |
| |
| <p>Once a user application is bundled, it can be launched using the <code class="language-plaintext highlighter-rouge">bin/spark-submit</code> script. |
| This script takes care of setting up the classpath with Spark and its |
| dependencies, and can support different cluster managers and deploy modes that Spark supports:</p> |
| |
| <figure class="highlight"><pre><code class="language-bash" data-lang="bash">./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> <main-class> <span class="se">\</span> |
| <span class="nt">--master</span> <master-url> <span class="se">\</span> |
| <span class="nt">--deploy-mode</span> <deploy-mode> <span class="se">\</span> |
| <span class="nt">--conf</span> <key><span class="o">=</span><value> <span class="se">\</span> |
| ... <span class="c"># other options</span> |
| <application-jar> <span class="se">\</span> |
| <span class="o">[</span>application-arguments]</code></pre></figure> |
| |
| <p>Some of the commonly used options are:</p> |
| |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">--class</code>: The entry point for your application (e.g. <code class="language-plaintext highlighter-rouge">org.apache.spark.examples.SparkPi</code>)</li> |
| <li><code class="language-plaintext highlighter-rouge">--master</code>: The <a href="#master-urls">master URL</a> for the cluster (e.g. <code class="language-plaintext highlighter-rouge">spark://23.195.26.187:7077</code>)</li> |
| <li><code class="language-plaintext highlighter-rouge">--deploy-mode</code>: Whether to deploy your driver on the worker nodes (<code class="language-plaintext highlighter-rouge">cluster</code>) or locally as an external client (<code class="language-plaintext highlighter-rouge">client</code>) (default: <code class="language-plaintext highlighter-rouge">client</code>) <b> † </b></li> |
| <li><code class="language-plaintext highlighter-rouge">--conf</code>: Arbitrary Spark configuration property in key=value format. For values that contain spaces wrap “key=value” in quotes (as shown). Multiple configurations should be passed as separate arguments. (e.g. <code class="language-plaintext highlighter-rouge">--conf <key>=<value> --conf <key2>=<value2></code>)</li> |
| <li><code class="language-plaintext highlighter-rouge">application-jar</code>: Path to a bundled jar including your application and all dependencies. The URL must be globally visible inside of your cluster, for instance, an <code class="language-plaintext highlighter-rouge">hdfs://</code> path or a <code class="language-plaintext highlighter-rouge">file://</code> path that is present on all nodes.</li> |
| <li><code class="language-plaintext highlighter-rouge">application-arguments</code>: Arguments passed to the main method of your main class, if any</li> |
| </ul> |
| |
| <p><b>†</b> A common deployment strategy is to submit your application from a gateway machine |
| that is |
| physically co-located with your worker machines (e.g. Master node in a standalone EC2 cluster). |
| In this setup, <code class="language-plaintext highlighter-rouge">client</code> mode is appropriate. In <code class="language-plaintext highlighter-rouge">client</code> mode, the driver is launched directly |
| within the <code class="language-plaintext highlighter-rouge">spark-submit</code> process which acts as a <em>client</em> to the cluster. The input and |
| output of the application is attached to the console. Thus, this mode is especially suitable |
| for applications that involve the REPL (e.g. Spark shell).</p> |
| |
| <p>Alternatively, if your application is submitted from a machine far from the worker machines (e.g. |
| locally on your laptop), it is common to use <code class="language-plaintext highlighter-rouge">cluster</code> mode to minimize network latency between |
| the drivers and the executors. Currently, the standalone mode does not support cluster mode for Python |
| applications.</p> |
| |
| <p>For Python applications, simply pass a <code class="language-plaintext highlighter-rouge">.py</code> file in the place of <code class="language-plaintext highlighter-rouge"><application-jar></code>, |
| and add Python <code class="language-plaintext highlighter-rouge">.zip</code>, <code class="language-plaintext highlighter-rouge">.egg</code> or <code class="language-plaintext highlighter-rouge">.py</code> files to the search path with <code class="language-plaintext highlighter-rouge">--py-files</code>.</p> |
| |
| <p>There are a few options available that are specific to the |
| <a href="cluster-overview.html#cluster-manager-types">cluster manager</a> that is being used. |
| For example, with a <a href="spark-standalone.html">Spark standalone cluster</a> with <code class="language-plaintext highlighter-rouge">cluster</code> deploy mode, |
| you can also specify <code class="language-plaintext highlighter-rouge">--supervise</code> to make sure that the driver is automatically restarted if it |
| fails with a non-zero exit code. To enumerate all such options available to <code class="language-plaintext highlighter-rouge">spark-submit</code>, |
| run it with <code class="language-plaintext highlighter-rouge">--help</code>. Here are a few examples of common options:</p> |
| |
| <figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="c"># Run application locally on 8 cores</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> <span class="nb">local</span><span class="o">[</span>8] <span class="se">\</span> |
| /path/to/examples.jar <span class="se">\</span> |
| 100 |
| |
| <span class="c"># Run on a Spark standalone cluster in client deploy mode</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> spark://207.184.161.138:7077 <span class="se">\</span> |
| <span class="nt">--executor-memory</span> 20G <span class="se">\</span> |
| <span class="nt">--total-executor-cores</span> 100 <span class="se">\</span> |
| /path/to/examples.jar <span class="se">\</span> |
| 1000 |
| |
| <span class="c"># Run on a Spark standalone cluster in cluster deploy mode with supervise</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> spark://207.184.161.138:7077 <span class="se">\</span> |
| <span class="nt">--deploy-mode</span> cluster <span class="se">\</span> |
| <span class="nt">--supervise</span> <span class="se">\</span> |
| <span class="nt">--executor-memory</span> 20G <span class="se">\</span> |
| <span class="nt">--total-executor-cores</span> 100 <span class="se">\</span> |
| /path/to/examples.jar <span class="se">\</span> |
| 1000 |
| |
| <span class="c"># Run on a YARN cluster in cluster deploy mode</span> |
| <span class="nb">export </span><span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>XXX |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> yarn <span class="se">\</span> |
| <span class="nt">--deploy-mode</span> cluster <span class="se">\</span> |
| <span class="nt">--executor-memory</span> 20G <span class="se">\</span> |
| <span class="nt">--num-executors</span> 50 <span class="se">\</span> |
| /path/to/examples.jar <span class="se">\</span> |
| 1000 |
| |
| <span class="c"># Run a Python application on a Spark standalone cluster</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--master</span> spark://207.184.161.138:7077 <span class="se">\</span> |
| examples/src/main/python/pi.py <span class="se">\</span> |
| 1000 |
| |
| <span class="c"># Run on a Mesos cluster in cluster deploy mode with supervise</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> mesos://207.184.161.138:7077 <span class="se">\</span> |
| <span class="nt">--deploy-mode</span> cluster <span class="se">\</span> |
| <span class="nt">--supervise</span> <span class="se">\</span> |
| <span class="nt">--executor-memory</span> 20G <span class="se">\</span> |
| <span class="nt">--total-executor-cores</span> 100 <span class="se">\</span> |
| http://path/to/examples.jar <span class="se">\</span> |
| 1000 |
| |
| <span class="c"># Run on a Kubernetes cluster in cluster deploy mode</span> |
| ./bin/spark-submit <span class="se">\</span> |
| <span class="nt">--class</span> org.apache.spark.examples.SparkPi <span class="se">\</span> |
| <span class="nt">--master</span> k8s://xx.yy.zz.ww:443 <span class="se">\</span> |
| <span class="nt">--deploy-mode</span> cluster <span class="se">\</span> |
| <span class="nt">--executor-memory</span> 20G <span class="se">\</span> |
| <span class="nt">--num-executors</span> 50 <span class="se">\</span> |
| http://path/to/examples.jar <span class="se">\</span> |
| 1000</code></pre></figure> |
| |
| <h1 id="master-urls">Master URLs</h1> |
| |
| <p>The master URL passed to Spark can be in one of the following formats:</p> |
| |
| <table class="table table-striped"> |
| <thead><tr><th>Master URL</th><th>Meaning</th></tr></thead> |
| <tr><td> <code>local</code> </td><td> Run Spark locally with one worker thread (i.e. no parallelism at all). </td></tr> |
| <tr><td> <code>local[K]</code> </td><td> Run Spark locally with K worker threads (ideally, set this to the number of cores on your machine). </td></tr> |
| <tr><td> <code>local[K,F]</code> </td><td> Run Spark locally with K worker threads and F maxFailures (see <a href="configuration.html#scheduling">spark.task.maxFailures</a> for an explanation of this variable). </td></tr> |
| <tr><td> <code>local[*]</code> </td><td> Run Spark locally with as many worker threads as logical cores on your machine.</td></tr> |
| <tr><td> <code>local[*,F]</code> </td><td> Run Spark locally with as many worker threads as logical cores on your machine and F maxFailures.</td></tr> |
| <tr><td> <code>local-cluster[N,C,M]</code> </td><td> Local-cluster mode is only for unit tests. It emulates a distributed cluster in a single JVM with N number of workers, C cores per worker and M MiB of memory per worker.</td></tr> |
| <tr><td> <code>spark://HOST:PORT</code> </td><td> Connect to the given <a href="spark-standalone.html">Spark standalone |
| cluster</a> master. The port must be whichever one your master is configured to use, which is 7077 by default. |
| </td></tr> |
| <tr><td> <code>spark://HOST1:PORT1,HOST2:PORT2</code> </td><td> Connect to the given <a href="spark-standalone.html#standby-masters-with-zookeeper">Spark standalone |
| cluster with standby masters with Zookeeper</a>. The list must have all the master hosts in the high availability cluster set up with Zookeeper. The port must be whichever each master is configured to use, which is 7077 by default. |
| </td></tr> |
| <tr><td> <code>mesos://HOST:PORT</code> </td><td> Connect to the given <a href="running-on-mesos.html">Mesos</a> cluster. |
| The port must be whichever one your is configured to use, which is 5050 by default. |
| Or, for a Mesos cluster using ZooKeeper, use <code>mesos://zk://...</code>. |
| To submit with <code>--deploy-mode cluster</code>, the HOST:PORT should be configured to connect to the <a href="running-on-mesos.html#cluster-mode">MesosClusterDispatcher</a>. |
| </td></tr> |
| <tr><td> <code>yarn</code> </td><td> Connect to a <a href="running-on-yarn.html"> YARN </a> cluster in |
| <code>client</code> or <code>cluster</code> mode depending on the value of <code>--deploy-mode</code>. |
| The cluster location will be found based on the <code>HADOOP_CONF_DIR</code> or <code>YARN_CONF_DIR</code> variable. |
| </td></tr> |
| <tr><td> <code>k8s://HOST:PORT</code> </td><td> Connect to a <a href="running-on-kubernetes.html">Kubernetes</a> cluster in |
| <code>client</code> or <code>cluster</code> mode depending on the value of <code>--deploy-mode</code>. |
| The <code>HOST</code> and <code>PORT</code> refer to the <a href="https://kubernetes.io/docs/reference/generated/kube-apiserver/">Kubernetes API Server</a>. |
| It connects using TLS by default. In order to force it to use an unsecured connection, you can use |
| <code>k8s://http://HOST:PORT</code>. |
| </td></tr> |
| </table> |
| |
| <h1 id="loading-configuration-from-a-file">Loading Configuration from a File</h1> |
| |
| <p>The <code class="language-plaintext highlighter-rouge">spark-submit</code> script can load default <a href="configuration.html">Spark configuration values</a> from a |
| properties file and pass them on to your application. By default, it will read options |
| from <code class="language-plaintext highlighter-rouge">conf/spark-defaults.conf</code> in the Spark directory. For more detail, see the section on |
| <a href="configuration.html#loading-default-configurations">loading default configurations</a>.</p> |
| |
| <p>Loading default Spark configurations this way can obviate the need for certain flags to |
| <code class="language-plaintext highlighter-rouge">spark-submit</code>. For instance, if the <code class="language-plaintext highlighter-rouge">spark.master</code> property is set, you can safely omit the |
| <code class="language-plaintext highlighter-rouge">--master</code> flag from <code class="language-plaintext highlighter-rouge">spark-submit</code>. In general, configuration values explicitly set on a |
| <code class="language-plaintext highlighter-rouge">SparkConf</code> take the highest precedence, then flags passed to <code class="language-plaintext highlighter-rouge">spark-submit</code>, then values in the |
| defaults file.</p> |
| |
| <p>If you are ever unclear where configuration options are coming from, you can print out fine-grained |
| debugging information by running <code class="language-plaintext highlighter-rouge">spark-submit</code> with the <code class="language-plaintext highlighter-rouge">--verbose</code> option.</p> |
| |
| <h1 id="advanced-dependency-management">Advanced Dependency Management</h1> |
| <p>When using <code class="language-plaintext highlighter-rouge">spark-submit</code>, the application jar along with any jars included with the <code class="language-plaintext highlighter-rouge">--jars</code> option |
| will be automatically transferred to the cluster. URLs supplied after <code class="language-plaintext highlighter-rouge">--jars</code> must be separated by commas. That list is included in the driver and executor classpaths. Directory expansion does not work with <code class="language-plaintext highlighter-rouge">--jars</code>.</p> |
| |
| <p>Spark uses the following URL scheme to allow different strategies for disseminating jars:</p> |
| |
| <ul> |
| <li><strong>file:</strong> - Absolute paths and <code class="language-plaintext highlighter-rouge">file:/</code> URIs are served by the driver’s HTTP file server, and |
| every executor pulls the file from the driver HTTP server.</li> |
| <li><strong>hdfs:</strong>, <strong>http:</strong>, <strong>https:</strong>, <strong>ftp:</strong> - these pull down files and JARs from the URI as expected</li> |
| <li><strong>local:</strong> - a URI starting with local:/ is expected to exist as a local file on each worker node. This |
| means that no network IO will be incurred, and works well for large files/JARs that are pushed to each worker, |
| or shared via NFS, GlusterFS, etc.</li> |
| </ul> |
| |
| <p>Note that JARs and files are copied to the working directory for each SparkContext on the executor nodes. |
| This can use up a significant amount of space over time and will need to be cleaned up. With YARN, cleanup |
| is handled automatically, and with Spark standalone, automatic cleanup can be configured with the |
| <code class="language-plaintext highlighter-rouge">spark.worker.cleanup.appDataTtl</code> property.</p> |
| |
| <p>Users may also include any other dependencies by supplying a comma-delimited list of Maven coordinates |
| with <code class="language-plaintext highlighter-rouge">--packages</code>. All transitive dependencies will be handled when using this command. Additional |
| repositories (or resolvers in SBT) can be added in a comma-delimited fashion with the flag <code class="language-plaintext highlighter-rouge">--repositories</code>. |
| (Note that credentials for password-protected repositories can be supplied in some cases in the repository URI, |
| such as in <code class="language-plaintext highlighter-rouge">https://user:password@host/...</code>. Be careful when supplying credentials this way.) |
| These commands can be used with <code class="language-plaintext highlighter-rouge">pyspark</code>, <code class="language-plaintext highlighter-rouge">spark-shell</code>, and <code class="language-plaintext highlighter-rouge">spark-submit</code> to include Spark Packages.</p> |
| |
| <p>For Python, the equivalent <code class="language-plaintext highlighter-rouge">--py-files</code> option can be used to distribute <code class="language-plaintext highlighter-rouge">.egg</code>, <code class="language-plaintext highlighter-rouge">.zip</code> and <code class="language-plaintext highlighter-rouge">.py</code> libraries |
| to executors.</p> |
| |
| <h1 id="more-information">More Information</h1> |
| |
| <p>Once you have deployed your application, the <a href="cluster-overview.html">cluster mode overview</a> describes |
| the components involved in distributed execution, and how to monitor and debug applications.</p> |
| |
| </div> |
| |
| <!-- /container --> |
| </div> |
| |
| <script src="js/vendor/jquery-3.5.1.min.js"></script> |
| <script src="js/vendor/bootstrap.bundle.min.js"></script> |
| |
| <script src="js/vendor/anchor.min.js"></script> |
| <script src="js/main.js"></script> |
| |
| <script type="text/javascript" src="js/vendor/docsearch.min.js"></script> |
| <script type="text/javascript"> |
| // DocSearch is entirely free and automated. DocSearch is built in two parts: |
| // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link |
| // in your website and extract content from every page it traverses. It then pushes this |
| // content to an Algolia index. |
| // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index |
| // to your search input and display its results in a dropdown UI. If you want to find more |
| // details on how works DocSearch, check the docs of DocSearch. |
| docsearch({ |
| apiKey: 'd62f962a82bc9abb53471cb7b89da35e', |
| appId: 'RAI69RXRSK', |
| indexName: 'apache_spark', |
| inputSelector: '#docsearch-input', |
| enhancedSearchInput: true, |
| algoliaOptions: { |
| 'facetFilters': ["version:3.5.0"] |
| }, |
| debug: false // Set debug to true if you want to inspect the dropdown |
| }); |
| |
| </script> |
| |
| <!-- MathJax Section --> |
| <script type="text/x-mathjax-config"> |
| MathJax.Hub.Config({ |
| TeX: { equationNumbers: { autoNumber: "AMS" } } |
| }); |
| </script> |
| <script> |
| // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS. |
| // We could use "//cdn.mathjax...", but that won't support "file://". |
| (function(d, script) { |
| script = d.createElement('script'); |
| script.type = 'text/javascript'; |
| script.async = true; |
| script.onload = function(){ |
| MathJax.Hub.Config({ |
| tex2jax: { |
| inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ], |
| displayMath: [ ["$$","$$"], ["\\[", "\\]"] ], |
| processEscapes: true, |
| skipTags: ['script', 'noscript', 'style', 'textarea', 'pre'] |
| } |
| }); |
| }; |
| script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + |
| 'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' + |
| '?config=TeX-AMS-MML_HTMLorMML'; |
| d.getElementsByTagName('head')[0].appendChild(script); |
| }(document)); |
| </script> |
| </body> |
| </html> |
