site/docs/3.5.3/sql-data-sources-text.html - spark-website - Git at Google


 <!DOCTYPE html>
 <!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
 <!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
 <!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
     <head>
         <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">

         <title>Text Files - Spark 3.5.3 Documentation</title>


         <link rel="stylesheet" href="css/bootstrap.min.css">
         <link rel="preconnect" href="https://fonts.googleapis.com">
         <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
         <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
         <link href="css/custom.css" rel="stylesheet">
         <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

         <link rel="stylesheet" href="css/pygments-default.css">
         <link rel="stylesheet" href="css/docsearch.min.css" />
         <link rel="stylesheet" href="css/docsearch.css">


         <!-- Matomo -->
         <script>
             var _paq = window._paq = window._paq || [];
             /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
             _paq.push(["disableCookies"]);
             _paq.push(['trackPageView']);
             _paq.push(['enableLinkTracking']);
             (function() {
               var u="https://analytics.apache.org/";
               _paq.push(['setTrackerUrl', u+'matomo.php']);
               _paq.push(['setSiteId', '40']);
               var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
               g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
             })();
         </script>
         <!-- End Matomo Code -->


     </head>
     <body class="global">
         <!--[if lt IE 7]>
             <p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
         <![endif]-->

         <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->

         <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar">
             <div class="navbar-brand"><a href="index.html">
                 <img src="img/spark-logo-rev.svg" width="141" height="72"/></a><span class="version">3.5.3</span>
             </div>
             <button class="navbar-toggler" type="button" data-toggle="collapse"
                     data-target="#navbarCollapse" aria-controls="navbarCollapse"
                     aria-expanded="false" aria-label="Toggle navigation">
                 <span class="navbar-toggler-icon"></span>
             </button>
             <div class="collapse navbar-collapse" id="navbarCollapse">
                 <ul class="navbar-nav me-auto">
                     <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
                         <div class="dropdown-menu" aria-labelledby="navbarQuickStart">
                             <a class="dropdown-item" href="quick-start.html">Quick Start</a>
                             <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
                             <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
                             <a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a>
                             <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
                             <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
                             <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
                             <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
                             <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
                         <div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
                             <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
                             <a class="dropdown-item" href="api/java/index.html">Java</a>
                             <a class="dropdown-item" href="api/python/index.html">Python</a>
                             <a class="dropdown-item" href="api/R/index.html">R</a>
                             <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
                         <div class="dropdown-menu" aria-labelledby="navbarDeploying">
                             <a class="dropdown-item" href="cluster-overview.html">Overview</a>
                             <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
                             <div class="dropdown-divider"></div>
                             <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
                             <a class="dropdown-item" href="running-on-mesos.html">Mesos</a>
                             <a class="dropdown-item" href="running-on-yarn.html">YARN</a>
                             <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
                         <div class="dropdown-menu" aria-labelledby="navbarMore">
                             <a class="dropdown-item" href="configuration.html">Configuration</a>
                             <a class="dropdown-item" href="monitoring.html">Monitoring</a>
                             <a class="dropdown-item" href="tuning.html">Tuning Guide</a>
                             <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
                             <a class="dropdown-item" href="security.html">Security</a>
                             <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
                             <a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
                             <div class="dropdown-divider"></div>
                             <a class="dropdown-item" href="building-spark.html">Building Spark</a>
                             <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
                             <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
                         </div>
                     </li>

                     <li class="nav-item">
                         <input type="text" id="docsearch-input" placeholder="Search the docs…">
                     </li>
                 </ul>
                 <!--<span class="navbar-text navbar-right"><span class="version-text">v3.5.3</span></span>-->
             </div>
         </nav>


         <div class="container">


                     <div class="left-menu-wrapper">
     <div class="left-menu">
         <h3><a href="sql-programming-guide.html">Spark SQL Guide</a></h3>

 <ul>

     <li>
         <a href="sql-getting-started.html">

                 Getting Started

         </a>
     </li>


     <li>
         <a href="sql-data-sources.html">

                 Data Sources

         </a>
     </li>


 <ul>

     <li>
         <a href="sql-data-sources-load-save-functions.html">

                 Generic Load/Save Functions

         </a>
     </li>


     <li>
         <a href="sql-data-sources-generic-options.html">

                 Generic File Source Options

         </a>
     </li>


     <li>
         <a href="sql-data-sources-parquet.html">

                 Parquet Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-orc.html">

                 ORC Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-json.html">

                 JSON Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-csv.html">

                 CSV Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-text.html">

                 Text Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-hive-tables.html">

                 Hive Tables

         </a>
     </li>


     <li>
         <a href="sql-data-sources-jdbc.html">

                 JDBC To Other Databases

         </a>
     </li>


     <li>
         <a href="sql-data-sources-avro.html">

                 Avro Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-protobuf.html">

                 Protobuf data

         </a>
     </li>


     <li>
         <a href="sql-data-sources-binaryFile.html">

                 Whole Binary Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-troubleshooting.html">

                 Troubleshooting

         </a>
     </li>


 </ul>


     <li>
         <a href="sql-performance-tuning.html">

                 Performance Tuning

         </a>
     </li>


     <li>
         <a href="sql-distributed-sql-engine.html">

                 Distributed SQL Engine

         </a>
     </li>


     <li>
         <a href="sql-pyspark-pandas-with-arrow.html">

                 PySpark Usage Guide for Pandas with Apache Arrow

         </a>
     </li>


     <li>
         <a href="sql-migration-guide.html">

                 Migration Guide

         </a>
     </li>


     <li>
         <a href="sql-ref.html">

                 SQL Reference

         </a>
     </li>


     <li>
         <a href="sql-error-conditions.html">

                 Error Conditions

         </a>
     </li>


 </ul>

     </div>
 </div>

                 <input id="nav-trigger" class="nav-trigger" checked type="checkbox">
                 <label for="nav-trigger"></label>
                 <div class="content-with-sidebar mr-3" id="content">

                         <h1 class="title">Text Files</h1>


                     <p>Spark SQL provides <code class="language-plaintext highlighter-rouge">spark.read().text("file_name")</code> to read a file or directory of text files into a Spark DataFrame, and <code class="language-plaintext highlighter-rouge">dataframe.write().text("path")</code> to write to a text file. When reading a text file, each line becomes each row that has string &#8220;value&#8221; column by default. The line separator can be changed as shown in the example below. The <code class="language-plaintext highlighter-rouge">option()</code> function can be used to customize the behavior of reading or writing, such as controlling behavior of the line separator, compression, and so on.</p>

 <div class="codetabs">

 <div data-lang="python">
     <div class="highlight"><pre class="codehilite"><code><span class="c1"># spark is from the previous example
 </span><span class="n">sc</span> <span class="o">=</span> <span class="n">spark</span><span class="p">.</span><span class="n">sparkContext</span>

 <span class="c1"># A text dataset is pointed to by path.
 # The path can be either a single text file or a directory of text files
 </span><span class="n">path</span> <span class="o">=</span> <span class="s">"examples/src/main/resources/people.txt"</span>

 <span class="n">df1</span> <span class="o">=</span> <span class="n">spark</span><span class="p">.</span><span class="n">read</span><span class="p">.</span><span class="n">text</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
 <span class="n">df1</span><span class="p">.</span><span class="n">show</span><span class="p">()</span>
 <span class="c1"># +-----------+
 # |      value|
 # +-----------+
 # |Michael, 29|
 # |   Andy, 30|
 # | Justin, 19|
 # +-----------+
 </span>
 <span class="c1"># You can use 'lineSep' option to define the line separator.
 # The line separator handles all `\r`, `\r\n` and `\n` by default.
 </span><span class="n">df2</span> <span class="o">=</span> <span class="n">spark</span><span class="p">.</span><span class="n">read</span><span class="p">.</span><span class="n">text</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">lineSep</span><span class="o">=</span><span class="s">","</span><span class="p">)</span>
 <span class="n">df2</span><span class="p">.</span><span class="n">show</span><span class="p">()</span>
 <span class="c1"># +-----------+
 # |      value|
 # +-----------+
 # |    Michael|
 # |   29\nAndy|
 # | 30\nJustin|
 # |       19\n|
 # +-----------+
 </span>
 <span class="c1"># You can also use 'wholetext' option to read each input file as a single row.
 </span><span class="n">df3</span> <span class="o">=</span> <span class="n">spark</span><span class="p">.</span><span class="n">read</span><span class="p">.</span><span class="n">text</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">wholetext</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
 <span class="n">df3</span><span class="p">.</span><span class="n">show</span><span class="p">()</span>
 <span class="c1"># +--------------------+
 # |               value|
 # +--------------------+
 # |Michael, 29\nAndy...|
 # +--------------------+
 </span>
 <span class="c1"># "output" is a folder which contains multiple text files and a _SUCCESS file.
 </span><span class="n">df1</span><span class="p">.</span><span class="n">write</span><span class="p">.</span><span class="n">csv</span><span class="p">(</span><span class="s">"output"</span><span class="p">)</span>

 <span class="c1"># You can specify the compression format using the 'compression' option.
 </span><span class="n">df1</span><span class="p">.</span><span class="n">write</span><span class="p">.</span><span class="n">text</span><span class="p">(</span><span class="s">"output_compressed"</span><span class="p">,</span> <span class="n">compression</span><span class="o">=</span><span class="s">"gzip"</span><span class="p">)</span></code></pre></div>
     <div><small>Find full example code at "examples/src/main/python/sql/datasource.py" in the Spark repo.</small></div>
   </div>

 <div data-lang="scala">
     <div class="highlight"><pre class="codehilite"><code><span class="c1">// A text dataset is pointed to by path.</span>
 <span class="c1">// The path can be either a single text file or a directory of text files</span>
 <span class="k">val</span> <span class="nv">path</span> <span class="k">=</span> <span class="s">"examples/src/main/resources/people.txt"</span>

 <span class="k">val</span> <span class="nv">df1</span> <span class="k">=</span> <span class="nv">spark</span><span class="o">.</span><span class="py">read</span><span class="o">.</span><span class="py">text</span><span class="o">(</span><span class="n">path</span><span class="o">)</span>
 <span class="nv">df1</span><span class="o">.</span><span class="py">show</span><span class="o">()</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |      value|</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |Michael, 29|</span>
 <span class="c1">// |   Andy, 30|</span>
 <span class="c1">// | Justin, 19|</span>
 <span class="c1">// +-----------+</span>

 <span class="c1">// You can use 'lineSep' option to define the line separator.</span>
 <span class="c1">// The line separator handles all `\r`, `\r\n` and `\n` by default.</span>
 <span class="k">val</span> <span class="nv">df2</span> <span class="k">=</span> <span class="nv">spark</span><span class="o">.</span><span class="py">read</span><span class="o">.</span><span class="py">option</span><span class="o">(</span><span class="s">"lineSep"</span><span class="o">,</span> <span class="s">","</span><span class="o">).</span><span class="py">text</span><span class="o">(</span><span class="n">path</span><span class="o">)</span>
 <span class="nv">df2</span><span class="o">.</span><span class="py">show</span><span class="o">()</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |      value|</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |    Michael|</span>
 <span class="c1">// |   29\nAndy|</span>
 <span class="c1">// | 30\nJustin|</span>
 <span class="c1">// |       19\n|</span>
 <span class="c1">// +-----------+</span>

 <span class="c1">// You can also use 'wholetext' option to read each input file as a single row.</span>
 <span class="k">val</span> <span class="nv">df3</span> <span class="k">=</span> <span class="nv">spark</span><span class="o">.</span><span class="py">read</span><span class="o">.</span><span class="py">option</span><span class="o">(</span><span class="s">"wholetext"</span><span class="o">,</span> <span class="kc">true</span><span class="o">).</span><span class="py">text</span><span class="o">(</span><span class="n">path</span><span class="o">)</span>
 <span class="nv">df3</span><span class="o">.</span><span class="py">show</span><span class="o">()</span>
 <span class="c1">//  +--------------------+</span>
 <span class="c1">//  |               value|</span>
 <span class="c1">//  +--------------------+</span>
 <span class="c1">//  |Michael, 29\nAndy...|</span>
 <span class="c1">//  +--------------------+</span>

 <span class="c1">// "output" is a folder which contains multiple text files and a _SUCCESS file.</span>
 <span class="nv">df1</span><span class="o">.</span><span class="py">write</span><span class="o">.</span><span class="py">text</span><span class="o">(</span><span class="s">"output"</span><span class="o">)</span>

 <span class="c1">// You can specify the compression format using the 'compression' option.</span>
 <span class="nv">df1</span><span class="o">.</span><span class="py">write</span><span class="o">.</span><span class="py">option</span><span class="o">(</span><span class="s">"compression"</span><span class="o">,</span> <span class="s">"gzip"</span><span class="o">).</span><span class="py">text</span><span class="o">(</span><span class="s">"output_compressed"</span><span class="o">)</span></code></pre></div>
     <div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala" in the Spark repo.</small></div>
   </div>

 <div data-lang="java">
     <div class="highlight"><pre class="codehilite"><code><span class="kn">import</span> <span class="nn">org.apache.spark.sql.Dataset</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.sql.Row</span><span class="o">;</span>

 <span class="c1">// A text dataset is pointed to by path.</span>
 <span class="c1">// The path can be either a single text file or a directory of text files</span>
 <span class="nc">String</span> <span class="n">path</span> <span class="o">=</span> <span class="s">"examples/src/main/resources/people.txt"</span><span class="o">;</span>

 <span class="nc">Dataset</span><span class="o">&lt;</span><span class="nc">Row</span><span class="o">&gt;</span> <span class="n">df1</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">text</span><span class="o">(</span><span class="n">path</span><span class="o">);</span>
 <span class="n">df1</span><span class="o">.</span><span class="na">show</span><span class="o">();</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |      value|</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |Michael, 29|</span>
 <span class="c1">// |   Andy, 30|</span>
 <span class="c1">// | Justin, 19|</span>
 <span class="c1">// +-----------+</span>

 <span class="c1">// You can use 'lineSep' option to define the line separator.</span>
 <span class="c1">// The line separator handles all `\r`, `\r\n` and `\n` by default.</span>
 <span class="nc">Dataset</span><span class="o">&lt;</span><span class="nc">Row</span><span class="o">&gt;</span> <span class="n">df2</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">option</span><span class="o">(</span><span class="s">"lineSep"</span><span class="o">,</span> <span class="s">","</span><span class="o">).</span><span class="na">text</span><span class="o">(</span><span class="n">path</span><span class="o">);</span>
 <span class="n">df2</span><span class="o">.</span><span class="na">show</span><span class="o">();</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |      value|</span>
 <span class="c1">// +-----------+</span>
 <span class="c1">// |    Michael|</span>
 <span class="c1">// |   29\nAndy|</span>
 <span class="c1">// | 30\nJustin|</span>
 <span class="c1">// |       19\n|</span>
 <span class="c1">// +-----------+</span>

 <span class="c1">// You can also use 'wholetext' option to read each input file as a single row.</span>
 <span class="nc">Dataset</span><span class="o">&lt;</span><span class="nc">Row</span><span class="o">&gt;</span> <span class="n">df3</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">option</span><span class="o">(</span><span class="s">"wholetext"</span><span class="o">,</span> <span class="s">"true"</span><span class="o">).</span><span class="na">text</span><span class="o">(</span><span class="n">path</span><span class="o">);</span>
 <span class="n">df3</span><span class="o">.</span><span class="na">show</span><span class="o">();</span>
 <span class="c1">//  +--------------------+</span>
 <span class="c1">//  |               value|</span>
 <span class="c1">//  +--------------------+</span>
 <span class="c1">//  |Michael, 29\nAndy...|</span>
 <span class="c1">//  +--------------------+</span>

 <span class="c1">// "output" is a folder which contains multiple text files and a _SUCCESS file.</span>
 <span class="n">df1</span><span class="o">.</span><span class="na">write</span><span class="o">().</span><span class="na">text</span><span class="o">(</span><span class="s">"output"</span><span class="o">);</span>

 <span class="c1">// You can specify the compression format using the 'compression' option.</span>
 <span class="n">df1</span><span class="o">.</span><span class="na">write</span><span class="o">().</span><span class="na">option</span><span class="o">(</span><span class="s">"compression"</span><span class="o">,</span> <span class="s">"gzip"</span><span class="o">).</span><span class="na">text</span><span class="o">(</span><span class="s">"output_compressed"</span><span class="o">);</span></code></pre></div>
     <div><small>Find full example code at "examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java" in the Spark repo.</small></div>
   </div>

 </div>

 <h2 id="data-source-option">Data Source Option</h2>

 <p>Data source options of text can be set via:</p>
 <ul>
   <li>the <code class="language-plaintext highlighter-rouge">.option</code>/<code class="language-plaintext highlighter-rouge">.options</code> methods of
     <ul>
       <li><code class="language-plaintext highlighter-rouge">DataFrameReader</code></li>
       <li><code class="language-plaintext highlighter-rouge">DataFrameWriter</code></li>
       <li><code class="language-plaintext highlighter-rouge">DataStreamReader</code></li>
       <li><code class="language-plaintext highlighter-rouge">DataStreamWriter</code></li>
     </ul>
   </li>
   <li><code class="language-plaintext highlighter-rouge">OPTIONS</code> clause at <a href="sql-ref-syntax-ddl-create-table-datasource.html">CREATE TABLE USING DATA_SOURCE</a></li>
 </ul>

 <table>
   <thead><tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Scope</b></th></tr></thead>
   <tr>
     <td><code>wholetext</code></td>
     <td><code>false</code></td>
     <td>If true, read each file from input path(s) as a single row.</td>
     <td>read</td>
   </tr>
   <tr>
     <td><code>lineSep</code></td>
     <td><code>\r</code>, <code>\r\n</code>, <code>\n</code> (for reading), <code>\n</code> (for writing)</td>
     <td>Defines the line separator that should be used for reading or writing.</td>
     <td>read/write</td>
   </tr>
   <tr>
     <td><code>compression</code></td>
     <td>(none)</td>
     <td>Compression codec to use when saving to file. This can be one of the known case-insensitive shorten names (none, bzip2, gzip, lz4, snappy and deflate).</td>
     <td>write</td>
   </tr>
 </table>
 <p>Other generic options can be found in <a href="https://spark.apache.org/docs/latest/sql-data-sources-generic-options.html"> Generic File Source Options</a>.</p>


                 </div>

              <!-- /container -->
         </div>

         <script src="js/vendor/jquery-3.5.1.min.js"></script>
         <script src="js/vendor/bootstrap.bundle.min.js"></script>

         <script src="js/vendor/anchor.min.js"></script>
         <script src="js/main.js"></script>

         <script type="text/javascript" src="js/vendor/docsearch.min.js"></script>
         <script type="text/javascript">
             // DocSearch is entirely free and automated. DocSearch is built in two parts:
             // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
             //    in your website and extract content from every page it traverses. It then pushes this
             //    content to an Algolia index.
             // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
             //    to your search input and display its results in a dropdown UI. If you want to find more
             //    details on how works DocSearch, check the docs of DocSearch.
             docsearch({
     apiKey: 'd62f962a82bc9abb53471cb7b89da35e',
     appId: 'RAI69RXRSK',
     indexName: 'apache_spark',
     inputSelector: '#docsearch-input',
     enhancedSearchInput: true,
     algoliaOptions: {
       'facetFilters': ["version:3.5.3"]
     },
     debug: false // Set debug to true if you want to inspect the dropdown
 });

         </script>

         <!-- MathJax Section -->
         <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 TeX: { equationNumbers: { autoNumber: "AMS" } }
             });
         </script>
         <script>
             // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
             // We could use "//cdn.mathjax...", but that won't support "file://".
             (function(d, script) {
                 script = d.createElement('script');
                 script.type = 'text/javascript';
                 script.async = true;
                 script.onload = function(){
                     MathJax.Hub.Config({
                         tex2jax: {
                             inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
                             displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
                             processEscapes: true,
                             skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
                         }
                     });
                 };
                 script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
                     'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
                     '?config=TeX-AMS-MML_HTMLorMML';
                 d.getElementsByTagName('head')[0].appendChild(script);
             }(document));
         </script>
     </body>
 </html>