Google Git
Sign in
apache / spark-website / 09eecfa321785be33d69343ae8327165c52e9973 / . / site / docs / 3.5.5 / api / python / _modules / pyspark / pandas / config.html
blob: c278ac79a872996f9e9bcd8d60f909830095fc6c [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>pyspark.pandas.config &#8212; PySpark 3.5.5 documentation</title>
<link href="../../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link rel="stylesheet"
href="../../../_static/vendor/fontawesome/5.13.0/css/all.min.css">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
<link rel="stylesheet" href="../../../_static/styles/pydata-sphinx-theme.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/css/pyspark.css" />
<link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf">
<script id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
<script src="../../../_static/jquery.js"></script>
<script src="../../../_static/underscore.js"></script>
<script src="../../../_static/doctools.js"></script>
<script src="../../../_static/language_data.js"></script>
<script src="../../../_static/clipboard.min.js"></script>
<script src="../../../_static/copybutton.js"></script>
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
<script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
<script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true, "ignoreClass": "document", "processClass": "math|output_area"}})</script>
<link rel="canonical" href="https://spark.apache.org/docs/latest/api/python/_modules/pyspark/pandas/config.html" />
<link rel="search" title="Search" href="../../../search.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
<!-- Google Analytics -->
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
<div class="container-fluid" id="banner"></div>
<nav class="navbar navbar-light navbar-expand-lg bg-light fixed-top bd-navbar" id="navbar-main"><div class="container-xl">
<div id="navbar-start">
<a class="navbar-brand" href="../../../index.html">
<img src="../../../_static/spark-logo-reverse.png" class="logo" alt="logo">
</a>
</div>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbar-collapsible" aria-controls="navbar-collapsible" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div id="navbar-collapsible" class="col-lg-9 collapse navbar-collapse">
<div id="navbar-center" class="mr-auto">
<div class="navbar-center-item">
<ul id="navbar-main-elements" class="navbar-nav">
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../index.html">
Overview
</a>
</li>
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../getting_started/index.html">
Getting Started
</a>
</li>
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../user_guide/index.html">
User Guides
</a>
</li>
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../reference/index.html">
API Reference
</a>
</li>
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../development/index.html">
Development
</a>
</li>
<li class="toctree-l1 nav-item">
<a class="reference internal nav-link" href="../../../migration_guide/index.html">
Migration Guides
</a>
</li>
</ul>
</div>
</div>
<div id="navbar-end">
<div class="navbar-end-item">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div id="version-button" class="dropdown">
<button type="button" class="btn btn-secondary btn-sm navbar-btn dropdown-toggle" id="version_switcher_button" data-toggle="dropdown">
3.5.5
<span class="caret"></span>
</button>
<div id="version_switcher" class="dropdown-menu list-group-flush py-0" aria-labelledby="version_switcher_button">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div>
<script type="text/javascript">
// Function to construct the target URL from the JSON components
function buildURL(entry) {
var template = "https://spark.apache.org/docs/{version}/api/python/index.html"; // supplied by jinja
template = template.replace("{version}", entry.version);
return template;
}
// Function to check if corresponding page path exists in other version of docs
// and, if so, go there instead of the homepage of the other docs version
function checkPageExistsAndRedirect(event) {
const currentFilePath = "_modules/pyspark/pandas/config.html",
otherDocsHomepage = event.target.getAttribute("href");
let tryUrl = `${otherDocsHomepage}${currentFilePath}`;
$.ajax({
type: 'HEAD',
url: tryUrl,
// if the page exists, go there
success: function() {
location.href = tryUrl;
}
}).fail(function() {
location.href = otherDocsHomepage;
});
return false;
}
// Function to populate the version switcher
(function () {
// get JSON config
$.getJSON("https://spark.apache.org/static/versions.json", function(data, textStatus, jqXHR) {
// create the nodes first (before AJAX calls) to ensure the order is
// correct (for now, links will go to doc version homepage)
$.each(data, function(index, entry) {
// if no custom name specified (e.g., "latest"), use version string
if (!("name" in entry)) {
entry.name = entry.version;
}
// construct the appropriate URL, and add it to the dropdown
entry.url = buildURL(entry);
const node = document.createElement("a");
node.setAttribute("class", "list-group-item list-group-item-action py-1");
node.setAttribute("href", `${entry.url}`);
node.textContent = `${entry.name}`;
node.onclick = checkPageExistsAndRedirect;
$("#version_switcher").append(node);
});
});
})();
</script>
</div>
</div>
</div>
</div>
</nav>
<div class="container-xl">
<div class="row">
<!-- Only show if we have sidebars configured, else just a small margin -->
<div class="col-12 col-md-3 bd-sidebar">
<div class="sidebar-start-items"><form class="bd-search d-flex align-items-center" action="../../../search.html" method="get">
<i class="icon fas fa-search"></i>
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" >
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
</div>
</nav>
</div>
<div class="sidebar-end-items">
</div>
</div>
<div class="d-none d-xl-block col-xl-2 bd-toc">
</div>
<main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main">
<div>
<h1>Source code for pyspark.pandas.config</h1><div class="highlight"><pre>
<span></span><span class="c1">#</span>
<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
<span class="c1"># contributor license agreements. See the NOTICE file distributed with</span>
<span class="c1"># this work for additional information regarding copyright ownership.</span>
<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
<span class="c1"># the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1">#</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Infrastructure of options for pandas-on-Spark.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">contextlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">contextmanager</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Callable</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">Iterator</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Union</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyspark._globals</span><span class="w"> </span><span class="kn">import</span> <span class="n">_NoValue</span><span class="p">,</span> <span class="n">_NoValueType</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyspark.pandas.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">default_session</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;get_option&quot;</span><span class="p">,</span> <span class="s2">&quot;set_option&quot;</span><span class="p">,</span> <span class="s2">&quot;reset_option&quot;</span><span class="p">,</span> <span class="s2">&quot;options&quot;</span><span class="p">,</span> <span class="s2">&quot;option_context&quot;</span><span class="p">]</span>
<span class="k">class</span><span class="w"> </span><span class="nc">Option</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Option class that defines an option with related properties.</span>
<span class="sd"> This class holds all information relevant to the one option. Also,</span>
<span class="sd"> Its instance can validate if the given value is acceptable or not.</span>
<span class="sd"> It is currently for internal usage only.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> key: str, keyword-only argument</span>
<span class="sd"> the option name to use.</span>
<span class="sd"> doc: str, keyword-only argument</span>
<span class="sd"> the documentation for the current option.</span>
<span class="sd"> default: Any, keyword-only argument</span>
<span class="sd"> default value for this option.</span>
<span class="sd"> types: Union[Tuple[type, ...], type], keyword-only argument</span>
<span class="sd"> default is str. It defines the expected types for this option. It is</span>
<span class="sd"> used with `isinstance` to validate the given value to this option.</span>
<span class="sd"> check_func: Tuple[Callable[[Any], bool], str], keyword-only argument</span>
<span class="sd"> default is a function that always returns `True` with an empty string.</span>
<span class="sd"> It defines:</span>
<span class="sd"> - a function to check the given value to this option</span>
<span class="sd"> - the error message to show when this check is failed</span>
<span class="sd"> When new value is set to this option, this function is called to check</span>
<span class="sd"> if the given value is valid.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> &gt;&gt;&gt; option = Option(</span>
<span class="sd"> ... key=&#39;option.name&#39;,</span>
<span class="sd"> ... doc=&quot;this is a test option&quot;,</span>
<span class="sd"> ... default=&quot;default&quot;,</span>
<span class="sd"> ... types=(float, int),</span>
<span class="sd"> ... check_func=(lambda v: v &gt; 0, &quot;should be a positive float&quot;))</span>
<span class="sd"> &gt;&gt;&gt; option.validate(&#39;abc&#39;) # doctest: +NORMALIZE_WHITESPACE</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> TypeError: The value for option &#39;option.name&#39; was &lt;class &#39;str&#39;&gt;;</span>
<span class="sd"> however, expected types are [(&lt;class &#39;float&#39;&gt;, &lt;class &#39;int&#39;&gt;)].</span>
<span class="sd"> &gt;&gt;&gt; option.validate(-1.1)</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> ValueError: should be a positive float</span>
<span class="sd"> &gt;&gt;&gt; option.validate(1.1)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="o">*</span><span class="p">,</span>
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">doc</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">default</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span>
<span class="n">types</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">type</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span> <span class="nb">type</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">check_func</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="n">Any</span><span class="p">],</span> <span class="nb">bool</span><span class="p">],</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span>
<span class="bp">self</span><span class="o">.</span><span class="n">doc</span> <span class="o">=</span> <span class="n">doc</span>
<span class="bp">self</span><span class="o">.</span><span class="n">default</span> <span class="o">=</span> <span class="n">default</span>
<span class="bp">self</span><span class="o">.</span><span class="n">types</span> <span class="o">=</span> <span class="n">types</span>
<span class="bp">self</span><span class="o">.</span><span class="n">check_func</span> <span class="o">=</span> <span class="n">check_func</span>
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Validate the given value and throw an exception with related information such as key.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">types</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
<span class="s2">&quot;The value for option &#39;</span><span class="si">%s</span><span class="s2">&#39; was </span><span class="si">%s</span><span class="s2">; however, expected types are &quot;</span>
<span class="s2">&quot;[</span><span class="si">%s</span><span class="s2">].&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">types</span><span class="p">))</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_func</span><span class="p">[</span><span class="mi">0</span><span class="p">](</span><span class="n">v</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">check_func</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
<span class="c1"># Available options.</span>
<span class="c1">#</span>
<span class="c1"># NOTE: if you are fixing or adding an option here, make sure you execute `show_options()` and</span>
<span class="c1"># copy &amp; paste the results into show_options</span>
<span class="c1"># &#39;docs/source/user_guide/pandas_on_spark/options.rst&#39; as well.</span>
<span class="c1"># See the examples below:</span>
<span class="c1"># &gt;&gt;&gt; from pyspark.pandas.config import show_options</span>
<span class="c1"># &gt;&gt;&gt; show_options()</span>
<span class="n">_options</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Option</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;display.max_rows&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;This sets the maximum number of rows pandas-on-Spark should output when printing out &quot;</span>
<span class="s2">&quot;various output. For example, this value determines the number of rows to be &quot;</span>
<span class="s2">&quot;shown at the repr() in a dataframe. Set `None` to unlimit the input length. &quot;</span>
<span class="s2">&quot;Default is 1000.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)),</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;display.max_rows&#39; should be greater than or equal to 0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.max_rows&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;compute.max_rows&#39; sets the limit of the current pandas-on-Spark DataFrame. &quot;</span>
<span class="s2">&quot;Set `None` to unlimit the input length. When the limit is set, it is executed &quot;</span>
<span class="s2">&quot;by the shortcut by collecting the data into the driver, and then using the pandas &quot;</span>
<span class="s2">&quot;API. If the limit is unset, the operation is executed by PySpark. Default is 1000.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)),</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;compute.max_rows&#39; should be greater than or equal to 0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.shortcut_limit&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;compute.shortcut_limit&#39; sets the limit for a shortcut. &quot;</span>
<span class="s2">&quot;It computes the specified number of rows and uses its schema. When the dataframe &quot;</span>
<span class="s2">&quot;length is larger than this limit, pandas-on-Spark uses PySpark to compute.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;compute.shortcut_limit&#39; should be greater than or equal to 0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.ops_on_diff_frames&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;This determines whether or not to operate between two different dataframes. &quot;</span>
<span class="s2">&quot;For example, &#39;combine_frames&#39; function internally performs a join operation which &quot;</span>
<span class="s2">&quot;can be expensive in general. So, if `compute.ops_on_diff_frames` variable is not &quot;</span>
<span class="s2">&quot;True, that method throws an exception.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">bool</span><span class="p">,</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.default_index_type&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span><span class="s2">&quot;This sets the default index type: sequence, distributed and distributed-sequence.&quot;</span><span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="s2">&quot;distributed-sequence&quot;</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;sequence&quot;</span><span class="p">,</span> <span class="s2">&quot;distributed&quot;</span><span class="p">,</span> <span class="s2">&quot;distributed-sequence&quot;</span><span class="p">),</span>
<span class="s2">&quot;Index type should be one of &#39;sequence&#39;, &#39;distributed&#39;, &#39;distributed-sequence&#39;.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.default_index_cache&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;This sets the default storage level for temporary RDDs cached in &quot;</span>
<span class="s2">&quot;distributed-sequence indexing: &#39;NONE&#39;, &#39;DISK_ONLY&#39;, &#39;DISK_ONLY_2&#39;, &quot;</span>
<span class="s2">&quot;&#39;DISK_ONLY_3&#39;, &#39;MEMORY_ONLY&#39;, &#39;MEMORY_ONLY_2&#39;, &#39;MEMORY_ONLY_SER&#39;, &quot;</span>
<span class="s2">&quot;&#39;MEMORY_ONLY_SER_2&#39;, &#39;MEMORY_AND_DISK&#39;, &#39;MEMORY_AND_DISK_2&#39;, &quot;</span>
<span class="s2">&quot;&#39;MEMORY_AND_DISK_SER&#39;, &#39;MEMORY_AND_DISK_SER_2&#39;, &#39;OFF_HEAP&#39;, &quot;</span>
<span class="s2">&quot;&#39;LOCAL_CHECKPOINT&#39;.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="s2">&quot;MEMORY_AND_DISK_SER&quot;</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span>
<span class="ow">in</span> <span class="p">(</span>
<span class="s2">&quot;NONE&quot;</span><span class="p">,</span>
<span class="s2">&quot;DISK_ONLY&quot;</span><span class="p">,</span>
<span class="s2">&quot;DISK_ONLY_2&quot;</span><span class="p">,</span>
<span class="s2">&quot;DISK_ONLY_3&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_ONLY&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_ONLY_2&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_ONLY_SER&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_ONLY_SER_2&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_AND_DISK&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_AND_DISK_2&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_AND_DISK_SER&quot;</span><span class="p">,</span>
<span class="s2">&quot;MEMORY_AND_DISK_SER_2&quot;</span><span class="p">,</span>
<span class="s2">&quot;OFF_HEAP&quot;</span><span class="p">,</span>
<span class="s2">&quot;LOCAL_CHECKPOINT&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="s2">&quot;Index type should be one of &#39;NONE&#39;, &#39;DISK_ONLY&#39;, &#39;DISK_ONLY_2&#39;, &quot;</span>
<span class="s2">&quot;&#39;DISK_ONLY_3&#39;, &#39;MEMORY_ONLY&#39;, &#39;MEMORY_ONLY_2&#39;, &#39;MEMORY_ONLY_SER&#39;, &quot;</span>
<span class="s2">&quot;&#39;MEMORY_ONLY_SER_2&#39;, &#39;MEMORY_AND_DISK&#39;, &#39;MEMORY_AND_DISK_2&#39;, &quot;</span>
<span class="s2">&quot;&#39;MEMORY_AND_DISK_SER&#39;, &#39;MEMORY_AND_DISK_SER_2&#39;, &#39;OFF_HEAP&#39;, &quot;</span>
<span class="s2">&quot;&#39;LOCAL_CHECKPOINT&#39;.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.ordered_head&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;compute.ordered_head&#39; sets whether or not to operate head with natural ordering. &quot;</span>
<span class="s2">&quot;pandas-on-Spark does not guarantee the row ordering so `head` could return some &quot;</span>
<span class="s2">&quot;rows from distributed partitions. If &#39;compute.ordered_head&#39; is set to True, &quot;</span>
<span class="s2">&quot;pandas-on-Spark performs natural ordering beforehand, but it will cause a &quot;</span>
<span class="s2">&quot;performance overhead.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">bool</span><span class="p">,</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.eager_check&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;compute.eager_check&#39; sets whether or not to launch some Spark jobs just for the sake &quot;</span>
<span class="s2">&quot;of validation. If &#39;compute.eager_check&#39; is set to True, pandas-on-Spark performs the &quot;</span>
<span class="s2">&quot;validation beforehand, but it will cause a performance overhead. Otherwise, &quot;</span>
<span class="s2">&quot;pandas-on-Spark skip the validation and will be slightly different from pandas. &quot;</span>
<span class="s2">&quot;Affected APIs: `Series.dot`, `Series.asof`, `Series.compare`, &quot;</span>
<span class="s2">&quot;`FractionalExtensionOps.astype`, `IntegralExtensionOps.astype`, &quot;</span>
<span class="s2">&quot;`FractionalOps.astype`, `DecimalOps.astype`, `skipna of statistical functions`.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">bool</span><span class="p">,</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;compute.isin_limit&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;compute.isin_limit&#39; sets the limit for filtering by &#39;Column.isin(list)&#39;. &quot;</span>
<span class="s2">&quot;If the length of the ‘list’ is above the limit, broadcast join is used instead &quot;</span>
<span class="s2">&quot;for better performance.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="mi">80</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;compute.isin_limit&#39; should be greater than or equal to 0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;plotting.max_rows&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;plotting.max_rows&#39; sets the visual limit on top-n-based plots such as `plot.bar` &quot;</span>
<span class="s2">&quot;and `plot.pie`. If it is set to 1000, the first 1000 data points will be used &quot;</span>
<span class="s2">&quot;for plotting. Default is 1000.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;plotting.max_rows&#39; should be greater than or equal to 0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;plotting.sample_ratio&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;&#39;plotting.sample_ratio&#39; sets the proportion of data that will be plotted for sample-&quot;</span>
<span class="s2">&quot;based plots such as `plot.line` and `plot.area`. &quot;</span>
<span class="s2">&quot;This option defaults to &#39;plotting.max_rows&#39; option.&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)),</span>
<span class="n">check_func</span><span class="o">=</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="mi">1</span> <span class="o">&gt;=</span> <span class="n">v</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span>
<span class="s2">&quot;&#39;plotting.sample_ratio&#39; should be 1.0 &gt;= value &gt;= 0.0.&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">),</span>
<span class="n">Option</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="s2">&quot;plotting.backend&quot;</span><span class="p">,</span>
<span class="n">doc</span><span class="o">=</span><span class="p">(</span>
<span class="s2">&quot;Backend to use for plotting. Default is plotly. &quot;</span>
<span class="s2">&quot;Supports any package that has a top-level `.plot` method. &quot;</span>
<span class="s2">&quot;Known options are: [matplotlib, plotly].&quot;</span>
<span class="p">),</span>
<span class="n">default</span><span class="o">=</span><span class="s2">&quot;plotly&quot;</span><span class="p">,</span>
<span class="n">types</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
<span class="p">),</span>
<span class="p">]</span>
<span class="n">_options_dict</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Option</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">((</span><span class="n">option</span><span class="o">.</span><span class="n">key</span> <span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">_options</span><span class="p">),</span> <span class="n">_options</span><span class="p">))</span>
<span class="n">_key_format</span> <span class="o">=</span> <span class="s2">&quot;pandas_on_Spark.</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span>
<span class="k">class</span><span class="w"> </span><span class="nc">OptionError</span><span class="p">(</span><span class="ne">AttributeError</span><span class="p">,</span> <span class="ne">KeyError</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">def</span><span class="w"> </span><span class="nf">show_options</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Make a pretty table that can be copied and pasted into public documentation.</span>
<span class="sd"> This is currently for an internal purpose.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> &gt;&gt;&gt; show_options() # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE</span>
<span class="sd"> ================... =======... =====================...</span>
<span class="sd"> Option Default Description</span>
<span class="sd"> ================... =======... =====================...</span>
<span class="sd"> display.max_rows 1000 This sets the maximum...</span>
<span class="sd"> ...</span>
<span class="sd"> ================... =======... =====================...</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">textwrap</span>
<span class="n">header</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Option&quot;</span><span class="p">,</span> <span class="s2">&quot;Default&quot;</span><span class="p">,</span> <span class="s2">&quot;Description&quot;</span><span class="p">]</span>
<span class="n">row_format</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{:&lt;31}</span><span class="s2"> </span><span class="si">{:&lt;23}</span><span class="s2"> </span><span class="si">{:&lt;53}</span><span class="s2">&quot;</span>
<span class="nb">print</span><span class="p">(</span><span class="n">row_format</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">31</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">23</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">53</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">row_format</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">*</span><span class="n">header</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">row_format</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">31</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">23</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">53</span><span class="p">))</span>
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">_options</span><span class="p">:</span>
<span class="n">doc</span> <span class="o">=</span> <span class="n">textwrap</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">option</span><span class="o">.</span><span class="n">doc</span><span class="p">,</span> <span class="mi">53</span><span class="p">)</span>
<span class="n">formatted</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">line</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="p">(</span><span class="s2">&quot; &quot;</span> <span class="o">*</span> <span class="mi">56</span><span class="p">)</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">doc</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)])</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="n">row_format</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">option</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="nb">repr</span><span class="p">(</span><span class="n">option</span><span class="o">.</span><span class="n">default</span><span class="p">),</span> <span class="n">formatted</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">row_format</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">31</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">23</span><span class="p">,</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="mi">53</span><span class="p">))</span>
<div class="viewcode-block" id="get_option"><a class="viewcode-back" href="../../../reference/pyspark.pandas/api/pyspark.pandas.get_option.html#pyspark.pandas.get_option">[docs]</a><span class="k">def</span><span class="w"> </span><span class="nf">get_option</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="n">_NoValueType</span><span class="p">]</span> <span class="o">=</span> <span class="n">_NoValue</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Retrieves the value of the specified option.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> key : str</span>
<span class="sd"> The key which should match a single option.</span>
<span class="sd"> default : object</span>
<span class="sd"> The default value if the option is not set yet. The value should be JSON serializable.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> result : the value of the option</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> OptionError : if no such option exists and the default is not provided</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_check_option</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">_NoValue</span><span class="p">:</span>
<span class="n">default</span> <span class="o">=</span> <span class="n">_options_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">default</span>
<span class="n">_options_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="n">default</span><span class="p">)</span>
<span class="n">spark_session</span> <span class="o">=</span> <span class="n">default_session</span><span class="p">()</span>
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">spark_session</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">_key_format</span><span class="p">(</span><span class="n">key</span><span class="p">),</span> <span class="n">default</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">default</span><span class="p">)))</span></div>
<div class="viewcode-block" id="set_option"><a class="viewcode-back" href="../../../reference/pyspark.pandas/api/pyspark.pandas.set_option.html#pyspark.pandas.set_option">[docs]</a><span class="k">def</span><span class="w"> </span><span class="nf">set_option</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Sets the value of the specified option.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> key : str</span>
<span class="sd"> The key which should match a single option.</span>
<span class="sd"> value : object</span>
<span class="sd"> New value of option. The value should be JSON serializable.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_check_option</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="n">_options_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="n">spark_session</span> <span class="o">=</span> <span class="n">default_session</span><span class="p">()</span>
<span class="n">spark_session</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">_key_format</span><span class="p">(</span><span class="n">key</span><span class="p">),</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">))</span></div>
<div class="viewcode-block" id="reset_option"><a class="viewcode-back" href="../../../reference/pyspark.pandas/api/pyspark.pandas.reset_option.html#pyspark.pandas.reset_option">[docs]</a><span class="k">def</span><span class="w"> </span><span class="nf">reset_option</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Reset one option to their default value.</span>
<span class="sd"> Pass &quot;all&quot; as an argument to reset all options.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> key : str</span>
<span class="sd"> If specified only option will be reset.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_check_option</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="n">default_session</span><span class="p">()</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">unset</span><span class="p">(</span><span class="n">_key_format</span><span class="p">(</span><span class="n">key</span><span class="p">))</span></div>
<div class="viewcode-block" id="option_context"><a class="viewcode-back" href="../../../reference/pyspark.pandas/api/pyspark.pandas.option_context.html#pyspark.pandas.option_context">[docs]</a><span class="nd">@contextmanager</span>
<span class="k">def</span><span class="w"> </span><span class="nf">option_context</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="kc">None</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Context manager to temporarily set options in the `with` statement context.</span>
<span class="sd"> You need to invoke ``option_context(pat, val, [(pat, val), ...])``.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> &gt;&gt;&gt; with option_context(&#39;display.max_rows&#39;, 10, &#39;compute.max_rows&#39;, 5):</span>
<span class="sd"> ... print(get_option(&#39;display.max_rows&#39;), get_option(&#39;compute.max_rows&#39;))</span>
<span class="sd"> 10 5</span>
<span class="sd"> &gt;&gt;&gt; print(get_option(&#39;display.max_rows&#39;), get_option(&#39;compute.max_rows&#39;))</span>
<span class="sd"> 1000 1000</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">%</span> <span class="mi">2</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Need to invoke as option_context(pat, val, [(pat, val), ...]).&quot;</span><span class="p">)</span>
<span class="n">opts</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">args</span><span class="p">[::</span><span class="mi">2</span><span class="p">],</span> <span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">::</span><span class="mi">2</span><span class="p">]))</span>
<span class="n">orig_opts</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">get_option</span><span class="p">(</span><span class="n">key</span><span class="p">)</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">opts</span><span class="p">}</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">opts</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">set_option</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
<span class="k">yield</span>
<span class="k">finally</span><span class="p">:</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">orig_opts</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">set_option</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_check_option</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_options_dict</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">OptionError</span><span class="p">(</span>
<span class="s2">&quot;No such option: &#39;</span><span class="si">{}</span><span class="s2">&#39;. Available options are [</span><span class="si">{}</span><span class="s2">]&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">key</span><span class="p">,</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">_options_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">class</span><span class="w"> </span><span class="nc">DictWrapper</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;provide attribute-style access to a nested dict&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Option</span><span class="p">],</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="p">):</span>
<span class="nb">object</span><span class="o">.</span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;d&quot;</span><span class="p">,</span> <span class="n">d</span><span class="p">)</span>
<span class="nb">object</span><span class="o">.</span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;prefix&quot;</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">val</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">prefix</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;prefix&quot;</span><span class="p">)</span>
<span class="n">d</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;d&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">prefix</span><span class="p">:</span>
<span class="n">prefix</span> <span class="o">+=</span> <span class="s2">&quot;.&quot;</span>
<span class="n">canonical_key</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">key</span>
<span class="n">candidates</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="n">x</span> <span class="ow">in</span> <span class="n">k</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">canonical_key</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">))</span>
<span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">candidates</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">candidates</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">canonical_key</span><span class="p">:</span>
<span class="n">set_option</span><span class="p">(</span><span class="n">canonical_key</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">OptionError</span><span class="p">(</span>
<span class="s2">&quot;No such option: &#39;</span><span class="si">{}</span><span class="s2">&#39;. Available options are [</span><span class="si">{}</span><span class="s2">]&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">key</span><span class="p">,</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">_options_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="s2">&quot;DictWrapper&quot;</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
<span class="n">prefix</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;prefix&quot;</span><span class="p">)</span>
<span class="n">d</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;d&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">prefix</span><span class="p">:</span>
<span class="n">prefix</span> <span class="o">+=</span> <span class="s2">&quot;.&quot;</span>
<span class="n">canonical_key</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">key</span>
<span class="n">candidates</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="n">x</span> <span class="ow">in</span> <span class="n">k</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">canonical_key</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">))</span>
<span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">candidates</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">candidates</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">canonical_key</span><span class="p">:</span>
<span class="k">return</span> <span class="n">get_option</span><span class="p">(</span><span class="n">canonical_key</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">candidates</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">OptionError</span><span class="p">(</span>
<span class="s2">&quot;No such option: &#39;</span><span class="si">{}</span><span class="s2">&#39;. Available options are [</span><span class="si">{}</span><span class="s2">]&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">key</span><span class="p">,</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">_options_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DictWrapper</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">canonical_key</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__dir__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="n">prefix</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;prefix&quot;</span><span class="p">)</span>
<span class="n">d</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;d&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">prefix</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="n">candidates</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
<span class="n">offset</span> <span class="o">=</span> <span class="mi">0</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">candidates</span> <span class="o">=</span> <span class="p">[</span><span class="n">k</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="n">x</span> <span class="ow">in</span> <span class="n">k</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">prefix</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">))]</span>
<span class="n">offset</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">prefix</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span> <span class="c1"># prefix (e.g. &quot;compute.&quot;) to trim.</span>
<span class="k">return</span> <span class="p">[</span><span class="n">c</span><span class="p">[</span><span class="n">offset</span><span class="p">:]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">candidates</span><span class="p">]</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">DictWrapper</span><span class="p">(</span><span class="n">_options_dict</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_test</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">doctest</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyspark.sql</span><span class="w"> </span><span class="kn">import</span> <span class="n">SparkSession</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pyspark.pandas.config</span>
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">&quot;SPARK_HOME&quot;</span><span class="p">])</span>
<span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">pandas</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">globs</span><span class="p">[</span><span class="s2">&quot;ps&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">pandas</span>
<span class="n">spark</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">&quot;local[4]&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">&quot;pyspark.pandas.config tests&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span>
<span class="p">)</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">(</span>
<span class="n">pyspark</span><span class="o">.</span><span class="n">pandas</span><span class="o">.</span><span class="n">config</span><span class="p">,</span>
<span class="n">globs</span><span class="o">=</span><span class="n">globs</span><span class="p">,</span>
<span class="n">optionflags</span><span class="o">=</span><span class="n">doctest</span><span class="o">.</span><span class="n">ELLIPSIS</span> <span class="o">|</span> <span class="n">doctest</span><span class="o">.</span><span class="n">NORMALIZE_WHITESPACE</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">spark</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;__main__&quot;</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
</div>
<!-- Previous / next buttons -->
<div class='prev-next-area'>
</div>
</main>
</div>
</div>
<script src="../../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script>
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p class="copyright">
&copy; Copyright .<br>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 3.0.4.<br>
</p>
</div>
</div>
</footer>
</body>
</html>