site/docs/4.1.0-preview1/api/python/migration_guide/index.html - spark-website - Git at Google



 <!DOCTYPE html>


 <html >

   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

     <title>Migration Guides &#8212; PySpark 4.1.0-preview1 documentation</title>


   <script data-cfasync="false">
     document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
     document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
   </script>

   <!-- Loaded before other Sphinx assets -->
   <link href="../_static/styles/theme.css?digest=e353d410970836974a52" rel="stylesheet" />
 <link href="../_static/styles/bootstrap.css?digest=e353d410970836974a52" rel="stylesheet" />
 <link href="../_static/styles/pydata-sphinx-theme.css?digest=e353d410970836974a52" rel="stylesheet" />


   <link href="../_static/vendor/fontawesome/6.1.2/css/all.min.css?digest=e353d410970836974a52" rel="stylesheet" />
   <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-solid-900.woff2" />
 <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-brands-400.woff2" />
 <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-regular-400.woff2" />

     <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" />
     <link rel="stylesheet" type="text/css" href="../_static/css/pyspark.css" />

   <!-- Pre-loaded scripts that we'll load fully later -->
   <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=e353d410970836974a52" />
 <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52" />

     <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
     <script src="../_static/jquery.js"></script>
     <script src="../_static/underscore.js"></script>
     <script src="../_static/doctools.js"></script>
     <script src="../_static/clipboard.min.js"></script>
     <script src="../_static/copybutton.js"></script>
     <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
     <script>DOCUMENTATION_OPTIONS.pagename = 'migration_guide/index';</script>
     <script>
         DOCUMENTATION_OPTIONS.theme_switcher_json_url = 'https://spark.apache.org/static/versions.json';
         DOCUMENTATION_OPTIONS.theme_switcher_version_match = '4.1.0-preview1';
         </script>
     <link rel="canonical" href="https://spark.apache.org/docs/latest/api/python/migration_guide/index.html" />
     <link rel="search" title="Search" href="../search.html" />
     <link rel="next" title="Upgrading PySpark" href="pyspark_upgrade.html" />
     <link rel="prev" title="Error classes in PySpark" href="../development/errors.html" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="docsearch:language" content="None">


     <!-- Matomo -->
     <script type="text/javascript">
         var _paq = window._paq = window._paq || [];
         /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
         _paq.push(["disableCookies"]);
         _paq.push(['trackPageView']);
         _paq.push(['enableLinkTracking']);
         (function() {
             var u="https://analytics.apache.org/";
             _paq.push(['setTrackerUrl', u+'matomo.php']);
             _paq.push(['setSiteId', '40']);
             var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
             g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
         })();
     </script>
     <!-- End Matomo Code -->

   </head>


   <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">


   <a class="skip-link" href="#main-content">Skip to main content</a>

   <input type="checkbox"
           class="sidebar-toggle"
           name="__primary"
           id="__primary"/>
   <label class="overlay overlay-primary" for="__primary"></label>

   <input type="checkbox"
           class="sidebar-toggle"
           name="__secondary"
           id="__secondary"/>
   <label class="overlay overlay-secondary" for="__secondary"></label>

   <div class="search-button__wrapper">
     <div class="search-button__overlay"></div>
     <div class="search-button__search-container">
 <form class="bd-search d-flex align-items-center"
       action="../search.html"
       method="get">
   <i class="fa-solid fa-magnifying-glass"></i>
   <input type="search"
          class="form-control"
          name="q"
          id="search-input"
          placeholder="Search the docs ..."
          aria-label="Search the docs ..."
          autocomplete="off"
          autocorrect="off"
          autocapitalize="off"
          spellcheck="false"/>
   <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
 </form></div>
   </div>

     <nav class="bd-header navbar navbar-expand-lg bd-navbar">
 <div class="bd-header__inner bd-page-width">
   <label class="sidebar-toggle primary-toggle" for="__primary">
     <span class="fa-solid fa-bars"></span>
   </label>

   <div class="navbar-header-items__start">

       <div class="navbar-item">


 <a class="navbar-brand logo" href="../index.html">


     <img src="https://spark.apache.org/images/spark-logo.png" class="logo__image only-light" alt="Logo image"/>
     <script>document.write(`<img src="https://spark.apache.org/images/spark-logo-rev.svg" class="logo__image only-dark" alt="Logo image"/>`);</script>


 </a></div>

   </div>


   <div class="col-lg-9 navbar-header-items">

     <div class="me-auto navbar-header-items__center">

         <div class="navbar-item"><nav class="navbar-nav">
   <p class="sidebar-header-items__title"
      role="heading"
      aria-level="1"
      aria-label="Site Navigation">
     Site Navigation
   </p>
   <ul class="bd-navbar-elements navbar-nav">

                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../index.html">
                         Overview
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../getting_started/index.html">
                         Getting Started
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../tutorial/index.html">
                         Tutorials
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../user_guide/index.html">
                         User Guide
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../reference/index.html">
                         API Reference
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../development/index.html">
                         Development
                       </a>
                     </li>

             <div class="nav-item dropdown">
                 <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                     More
                 </button>
                 <div class="dropdown-menu">

                     <li class="nav-item current active">
                       <a class="nav-link nav-internal" href="#">
                         Migration Guides
                       </a>
                     </li>

                 </div>
             </div>

   </ul>
 </nav></div>

     </div>


     <div class="navbar-header-items__end">

         <div class="navbar-item navbar-persistent--container">

 <script>
 document.write(`
   <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <i class="fa-solid fa-magnifying-glass"></i>
   </button>
 `);
 </script>
         </div>


         <div class="navbar-item">
 <script>
 document.write(`
   <div class="version-switcher__container dropdown">
     <button type="button" class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" data-bs-toggle="dropdown">
       4.1.0-preview1  <!-- this text may get changed later by javascript -->
       <span class="caret"></span>
     </button>
     <div class="version-switcher__menu dropdown-menu list-group-flush py-0">
     <!-- dropdown will be populated by javascript on page load -->
     </div>
   </div>
 `);
 </script></div>

         <div class="navbar-item">
 <script>
 document.write(`
   <button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
     <span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
     <span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
   </button>
 `);
 </script></div>

         <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
             <label class="sr-only">GitHub</label></a>
         </li>
         <li class="nav-item">


           <a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
             <label class="sr-only">PyPI</label></a>
         </li>
 </ul></div>

     </div>

   </div>


     <div class="navbar-persistent--mobile">
 <script>
 document.write(`
   <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <i class="fa-solid fa-magnifying-glass"></i>
   </button>
 `);
 </script>
     </div>


     <label class="sidebar-toggle secondary-toggle" for="__secondary">
       <span class="fa-solid fa-outdent"></span>
     </label>

 </div>

     </nav>

   <div class="bd-container">
     <div class="bd-container__inner bd-page-width">

       <div class="bd-sidebar-primary bd-sidebar">


   <div class="sidebar-header-items sidebar-primary__section">


       <div class="sidebar-header-items__center">

           <div class="navbar-item"><nav class="navbar-nav">
   <p class="sidebar-header-items__title"
      role="heading"
      aria-level="1"
      aria-label="Site Navigation">
     Site Navigation
   </p>
   <ul class="bd-navbar-elements navbar-nav">

                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../index.html">
                         Overview
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../getting_started/index.html">
                         Getting Started
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../tutorial/index.html">
                         Tutorials
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../user_guide/index.html">
                         User Guide
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../reference/index.html">
                         API Reference
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../development/index.html">
                         Development
                       </a>
                     </li>

             <div class="nav-item dropdown">
                 <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                     More
                 </button>
                 <div class="dropdown-menu">

                     <li class="nav-item current active">
                       <a class="nav-link nav-internal" href="#">
                         Migration Guides
                       </a>
                     </li>

                 </div>
             </div>

   </ul>
 </nav></div>

       </div>


       <div class="sidebar-header-items__end">

           <div class="navbar-item">
 <script>
 document.write(`
   <div class="version-switcher__container dropdown">
     <button type="button" class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" data-bs-toggle="dropdown">
       4.1.0-preview1  <!-- this text may get changed later by javascript -->
       <span class="caret"></span>
     </button>
     <div class="version-switcher__menu dropdown-menu list-group-flush py-0">
     <!-- dropdown will be populated by javascript on page load -->
     </div>
   </div>
 `);
 </script></div>

           <div class="navbar-item">
 <script>
 document.write(`
   <button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
     <span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
     <span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
   </button>
 `);
 </script></div>

           <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
             <label class="sr-only">GitHub</label></a>
         </li>
         <li class="nav-item">


           <a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
             <label class="sr-only">PyPI</label></a>
         </li>
 </ul></div>

       </div>

   </div>

     <div class="sidebar-primary-items__start sidebar-primary__section">
         <div class="sidebar-primary-item"><nav class="bd-docs-nav bd-links"
      aria-label="Section Navigation">
   <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
   <div class="bd-toc-item navbar-nav"><ul class="nav bd-sidenav">
 <li class="toctree-l1"><a class="reference internal" href="pyspark_upgrade.html">Upgrading PySpark</a></li>
 </ul>
 <ul class="nav bd-sidenav">
 <li class="toctree-l1"><a class="reference internal" href="koalas_to_pyspark.html">Migrating from Koalas to pandas API on Spark</a></li>
 </ul>
 </div>
 </nav></div>
     </div>


   <div class="sidebar-primary-items__end sidebar-primary__section">
   </div>

   <div id="rtd-footer-container"></div>


       </div>

       <main id="main-content" class="bd-main">


           <div class="bd-content">
             <div class="bd-article-container">

               <div class="bd-header-article">
 <div class="header-article-items header-article__inner">

     <div class="header-article-items__start">

         <div class="header-article-item">


 <nav aria-label="Breadcrumbs">
   <ul class="bd-breadcrumbs" role="navigation" aria-label="Breadcrumb">

     <li class="breadcrumb-item breadcrumb-home">
       <a href="../index.html" class="nav-link" aria-label="Home">
         <i class="fa-solid fa-home"></i>
       </a>
     </li>
     <li class="breadcrumb-item active" aria-current="page">Migration Guides</li>
   </ul>
 </nav>
 </div>

     </div>


 </div>
 </div>


 <div id="searchbox"></div>
                 <article class="bd-article" role="main">

   <section id="migration-guides">
 <h1>Migration Guides<a class="headerlink" href="#migration-guides" title="Permalink to this headline">#</a></h1>
 <p>This page includes links to guides that will help you migrate to PySpark.</p>
 <p>If you are upgrading from an older to a newer version of PySpark, refer to
 the following page for differences between versions:</p>
 <ul class="simple">
 <li><p><a class="reference internal" href="pyspark_upgrade.html"><span class="std std-ref">Upgrading PySpark</span></a></p></li>
 </ul>
 <div class="toctree-wrapper compound">
 </div>
 <p>If you are migrating code from <a class="reference external" href="https://koalas.readthedocs.io/en/latest">Koalas</a> to PySpark,
 this guide is for you:</p>
 <div class="toctree-wrapper compound">
 <ul>
 <li class="toctree-l1"><a class="reference internal" href="koalas_to_pyspark.html">Migrating from Koalas to pandas API on Spark</a></li>
 </ul>
 </div>
 <p>A lot of content in other migration guides can also be helpful when migrating to newer PySpark versions because PySpark internally shares the same components.
 Please also refer to the following migration guides:</p>
 <ul class="simple">
 <li><p><a class="reference external" href="https://spark.apache.org/docs/latest/sql-migration-guide.html">Migration Guide: SQL, Datasets and DataFrame</a></p></li>
 <li><p><a class="reference external" href="https://spark.apache.org/docs/latest/ss-migration-guide.html">Migration Guide: Structured Streaming</a></p></li>
 <li><p><a class="reference external" href="https://spark.apache.org/docs/latest/ml-migration-guide.html">Migration Guide: MLlib (Machine Learning)</a></p></li>
 <li><p><a class="reference external" href="https://spark.apache.org/docs/latest/core-migration-guide.html">Migration Guide: Spark Core</a></p></li>
 </ul>
 </section>


                 </article>


                 <footer class="bd-footer-article">

 <div class="footer-article-items footer-article__inner">

     <div class="footer-article-item"><!-- Previous / next buttons -->
 <div class="prev-next-area">
     <a class="left-prev"
        href="../development/errors.html"
        title="previous page">
       <i class="fa-solid fa-angle-left"></i>
       <div class="prev-next-info">
         <p class="prev-next-subtitle">previous</p>
         <p class="prev-next-title">Error classes in PySpark</p>
       </div>
     </a>
     <a class="right-next"
        href="pyspark_upgrade.html"
        title="next page">
       <div class="prev-next-info">
         <p class="prev-next-subtitle">next</p>
         <p class="prev-next-title">Upgrading PySpark</p>
       </div>
       <i class="fa-solid fa-angle-right"></i>
     </a>
 </div></div>

 </div>

                 </footer>

             </div>


                 <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">

   <div class="sidebar-secondary-item">
   <div class="tocsection sourcelink">
     <a href="../_sources/migration_guide/index.rst.txt">
       <i class="fa-solid fa-file-lines"></i> Show Source
     </a>
   </div>
 </div>

 </div></div>


           </div>
           <footer class="bd-footer-content">

           </footer>

       </main>
     </div>
   </div>

   <!-- Scripts loaded after <body> so the DOM is not blocked -->
   <script src="../_static/scripts/bootstrap.js?digest=e353d410970836974a52"></script>
 <script src="../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52"></script>

   <footer class="bd-footer">
 <div class="bd-footer__inner bd-page-width">

     <div class="footer-items__start">

         <div class="footer-item"><p class="copyright">
     Copyright @ 2025 The Apache Software Foundation, Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
 </p></div>

         <div class="footer-item">
   <p class="sphinx-version">
     Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 4.5.0.
     <br/>
   </p>
 </div>

     </div>


     <div class="footer-items__end">

         <div class="footer-item"><p class="theme-version">
   Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.13.3.
 </p></div>

     </div>

 </div>

   </footer>
   </body>
 </html>


	<!DOCTYPE html>


	<html >

	<head>
	<meta charset="utf-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

	<title>Migration Guides — PySpark 4.1.0-preview1 documentation</title>



	<script data-cfasync="false">
	document.documentElement.dataset.mode = localStorage.getItem("mode") \|\| "";
	document.documentElement.dataset.theme = localStorage.getItem("theme") \|\| "light";
	</script>

	<!-- Loaded before other Sphinx assets -->
	<link href="../_static/styles/theme.css?digest=e353d410970836974a52" rel="stylesheet" />
	<link href="../_static/styles/bootstrap.css?digest=e353d410970836974a52" rel="stylesheet" />
	<link href="../_static/styles/pydata-sphinx-theme.css?digest=e353d410970836974a52" rel="stylesheet" />


	<link href="../_static/vendor/fontawesome/6.1.2/css/all.min.css?digest=e353d410970836974a52" rel="stylesheet" />
	<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-solid-900.woff2" />
	<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-brands-400.woff2" />
	<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.1.2/webfonts/fa-regular-400.woff2" />

	<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
	<link rel="stylesheet" type="text/css" href="../_static/copybutton.css" />
	<link rel="stylesheet" type="text/css" href="../_static/css/pyspark.css" />

	<!-- Pre-loaded scripts that we'll load fully later -->
	<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=e353d410970836974a52" />
	<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52" />

	<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
	<script src="../_static/jquery.js"></script>
	<script src="../_static/underscore.js"></script>
	<script src="../_static/doctools.js"></script>
	<script src="../_static/clipboard.min.js"></script>
	<script src="../_static/copybutton.js"></script>
	<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
	<script>DOCUMENTATION_OPTIONS.pagename = 'migration_guide/index';</script>
	<script>
	DOCUMENTATION_OPTIONS.theme_switcher_json_url = 'https://spark.apache.org/static/versions.json';
	DOCUMENTATION_OPTIONS.theme_switcher_version_match = '4.1.0-preview1';
	</script>
	<link rel="canonical" href="https://spark.apache.org/docs/latest/api/python/migration_guide/index.html" />
	<link rel="search" title="Search" href="../search.html" />
	<link rel="next" title="Upgrading PySpark" href="pyspark_upgrade.html" />
	<link rel="prev" title="Error classes in PySpark" href="../development/errors.html" />
	<meta name="viewport" content="width=device-width, initial-scale=1" />
	<meta name="docsearch:language" content="None">


	<!-- Matomo -->
	<script type="text/javascript">
	var _paq = window._paq = window._paq \|\| [];
	/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
	_paq.push(["disableCookies"]);
	_paq.push(['trackPageView']);
	_paq.push(['enableLinkTracking']);
	(function() {
	var u="https://analytics.apache.org/";
	_paq.push(['setTrackerUrl', u+'matomo.php']);
	_paq.push(['setSiteId', '40']);
	var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
	g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
	})();
	</script>
	<!-- End Matomo Code -->

	</head>


	<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">



	<a class="skip-link" href="#main-content">Skip to main content</a>

	<input type="checkbox"
	class="sidebar-toggle"
	name="__primary"
	id="__primary"/>
	<label class="overlay overlay-primary" for="__primary"></label>

	<input type="checkbox"
	class="sidebar-toggle"
	name="__secondary"
	id="__secondary"/>
	<label class="overlay overlay-secondary" for="__secondary"></label>

	<div class="search-button__wrapper">
	<div class="search-button__overlay"></div>
	<div class="search-button__search-container">
	<form class="bd-search d-flex align-items-center"
	action="../search.html"
	method="get">
	<i class="fa-solid fa-magnifying-glass"></i>
	<input type="search"
	class="form-control"
	name="q"
	id="search-input"
	placeholder="Search the docs ..."
	aria-label="Search the docs ..."
	autocomplete="off"
	autocorrect="off"
	autocapitalize="off"
	spellcheck="false"/>
	<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
	</form></div>
	</div>

	<nav class="bd-header navbar navbar-expand-lg bd-navbar">
	<div class="bd-header__inner bd-page-width">
	<label class="sidebar-toggle primary-toggle" for="__primary">
	<span class="fa-solid fa-bars"></span>
	</label>

	<div class="navbar-header-items__start">

	<div class="navbar-item">


	<a class="navbar-brand logo" href="../index.html">









	<img src="https://spark.apache.org/images/spark-logo.png" class="logo__image only-light" alt="Logo image"/>
	<script>document.write(`<img src="https://spark.apache.org/images/spark-logo-rev.svg" class="logo__image only-dark" alt="Logo image"/>`);</script>


	</a></div>

	</div>


	<div class="col-lg-9 navbar-header-items">

	<div class="me-auto navbar-header-items__center">

	<div class="navbar-item"><nav class="navbar-nav">
	<p class="sidebar-header-items__title"
	role="heading"
	aria-level="1"
	aria-label="Site Navigation">
	Site Navigation
	</p>
	<ul class="bd-navbar-elements navbar-nav">

	<li class="nav-item">
	<a class="nav-link nav-internal" href="../index.html">
	Overview
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../getting_started/index.html">
	Getting Started
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../tutorial/index.html">
	Tutorials
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../user_guide/index.html">
	User Guide
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../reference/index.html">
	API Reference
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../development/index.html">
	Development
	</a>
	</li>

	<div class="nav-item dropdown">
	<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
	More
	</button>
	<div class="dropdown-menu">

	<li class="nav-item current active">
	<a class="nav-link nav-internal" href="#">
	Migration Guides
	</a>
	</li>

	</div>
	</div>

	</ul>
	</nav></div>

	</div>


	<div class="navbar-header-items__end">

	<div class="navbar-item navbar-persistent--container">

	<script>
	document.write(`
	<button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
	<i class="fa-solid fa-magnifying-glass"></i>
	</button>
	`);
	</script>
	</div>


	<div class="navbar-item">
	<script>
	document.write(`
	<div class="version-switcher__container dropdown">
	<button type="button" class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" data-bs-toggle="dropdown">
	4.1.0-preview1 <!-- this text may get changed later by javascript -->
	<span class="caret"></span>
	</button>
	<div class="version-switcher__menu dropdown-menu list-group-flush py-0">
	<!-- dropdown will be populated by javascript on page load -->
	</div>
	</div>
	`);
	</script></div>

	<div class="navbar-item">
	<script>
	document.write(`
	<button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
	<span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
	<span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
	<span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
	</button>
	`);
	</script></div>

	<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
	aria-label="Icon Links">
	<li class="nav-item">








	<a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
	<label class="sr-only">GitHub</label></a>
	</li>
	<li class="nav-item">








	<a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
	<label class="sr-only">PyPI</label></a>
	</li>
	</ul></div>

	</div>

	</div>


	<div class="navbar-persistent--mobile">
	<script>
	document.write(`
	<button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
	<i class="fa-solid fa-magnifying-glass"></i>
	</button>
	`);
	</script>
	</div>



	<label class="sidebar-toggle secondary-toggle" for="__secondary">
	<span class="fa-solid fa-outdent"></span>
	</label>

	</div>

	</nav>

	<div class="bd-container">
	<div class="bd-container__inner bd-page-width">

	<div class="bd-sidebar-primary bd-sidebar">



	<div class="sidebar-header-items sidebar-primary__section">


	<div class="sidebar-header-items__center">

	<div class="navbar-item"><nav class="navbar-nav">
	<p class="sidebar-header-items__title"
	role="heading"
	aria-level="1"
	aria-label="Site Navigation">
	Site Navigation
	</p>
	<ul class="bd-navbar-elements navbar-nav">

	<li class="nav-item">
	<a class="nav-link nav-internal" href="../index.html">
	Overview
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../getting_started/index.html">
	Getting Started
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../tutorial/index.html">
	Tutorials
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../user_guide/index.html">
	User Guide
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../reference/index.html">
	API Reference
	</a>
	</li>


	<li class="nav-item">
	<a class="nav-link nav-internal" href="../development/index.html">
	Development
	</a>
	</li>

	<div class="nav-item dropdown">
	<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
	More
	</button>
	<div class="dropdown-menu">

	<li class="nav-item current active">
	<a class="nav-link nav-internal" href="#">
	Migration Guides
	</a>
	</li>

	</div>
	</div>

	</ul>
	</nav></div>

	</div>



	<div class="sidebar-header-items__end">

	<div class="navbar-item">
	<script>
	document.write(`
	<div class="version-switcher__container dropdown">
	<button type="button" class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" data-bs-toggle="dropdown">
	4.1.0-preview1 <!-- this text may get changed later by javascript -->
	<span class="caret"></span>
	</button>
	<div class="version-switcher__menu dropdown-menu list-group-flush py-0">
	<!-- dropdown will be populated by javascript on page load -->
	</div>
	</div>
	`);
	</script></div>

	<div class="navbar-item">
	<script>
	document.write(`
	<button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
	<span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
	<span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
	<span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
	</button>
	`);
	</script></div>

	<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
	aria-label="Icon Links">
	<li class="nav-item">








	<a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
	<label class="sr-only">GitHub</label></a>
	</li>
	<li class="nav-item">








	<a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
	<label class="sr-only">PyPI</label></a>
	</li>
	</ul></div>

	</div>

	</div>

	<div class="sidebar-primary-items__start sidebar-primary__section">
	<div class="sidebar-primary-item"><nav class="bd-docs-nav bd-links"
	aria-label="Section Navigation">
	<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
	<div class="bd-toc-item navbar-nav"><ul class="nav bd-sidenav">
	<li class="toctree-l1"><a class="reference internal" href="pyspark_upgrade.html">Upgrading PySpark</a></li>
	</ul>
	<ul class="nav bd-sidenav">
	<li class="toctree-l1"><a class="reference internal" href="koalas_to_pyspark.html">Migrating from Koalas to pandas API on Spark</a></li>
	</ul>
	</div>
	</nav></div>
	</div>


	<div class="sidebar-primary-items__end sidebar-primary__section">
	</div>

	<div id="rtd-footer-container"></div>


	</div>

	<main id="main-content" class="bd-main">


	<div class="bd-content">
	<div class="bd-article-container">

	<div class="bd-header-article">
	<div class="header-article-items header-article__inner">

	<div class="header-article-items__start">

	<div class="header-article-item">



	<nav aria-label="Breadcrumbs">
	<ul class="bd-breadcrumbs" role="navigation" aria-label="Breadcrumb">

	<li class="breadcrumb-item breadcrumb-home">
	<a href="../index.html" class="nav-link" aria-label="Home">
	<i class="fa-solid fa-home"></i>
	</a>
	</li>
	<li class="breadcrumb-item active" aria-current="page">Migration Guides</li>
	</ul>
	</nav>
	</div>

	</div>


	</div>
	</div>




	<div id="searchbox"></div>
	<article class="bd-article" role="main">

	<section id="migration-guides">
	<h1>Migration Guides<a class="headerlink" href="#migration-guides" title="Permalink to this headline">#</a></h1>
	<p>This page includes links to guides that will help you migrate to PySpark.</p>
	<p>If you are upgrading from an older to a newer version of PySpark, refer to
	the following page for differences between versions:</p>
	<ul class="simple">
	<li><p><a class="reference internal" href="pyspark_upgrade.html"><span class="std std-ref">Upgrading PySpark</span></a></p></li>
	</ul>
	<div class="toctree-wrapper compound">
	</div>
	<p>If you are migrating code from <a class="reference external" href="https://koalas.readthedocs.io/en/latest">Koalas</a> to PySpark,
	this guide is for you:</p>
	<div class="toctree-wrapper compound">
	<ul>
	<li class="toctree-l1"><a class="reference internal" href="koalas_to_pyspark.html">Migrating from Koalas to pandas API on Spark</a></li>
	</ul>
	</div>
	<p>A lot of content in other migration guides can also be helpful when migrating to newer PySpark versions because PySpark internally shares the same components.
	Please also refer to the following migration guides:</p>
	<ul class="simple">
	<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/sql-migration-guide.html">Migration Guide: SQL, Datasets and DataFrame</a></p></li>
	<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/ss-migration-guide.html">Migration Guide: Structured Streaming</a></p></li>
	<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/ml-migration-guide.html">Migration Guide: MLlib (Machine Learning)</a></p></li>
	<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/core-migration-guide.html">Migration Guide: Spark Core</a></p></li>
	</ul>
	</section>


	</article>



	<footer class="bd-footer-article">

	<div class="footer-article-items footer-article__inner">

	<div class="footer-article-item"><!-- Previous / next buttons -->
	<div class="prev-next-area">
	<a class="left-prev"
	href="../development/errors.html"
	title="previous page">
	<i class="fa-solid fa-angle-left"></i>
	<div class="prev-next-info">
	<p class="prev-next-subtitle">previous</p>
	<p class="prev-next-title">Error classes in PySpark</p>
	</div>
	</a>
	<a class="right-next"
	href="pyspark_upgrade.html"
	title="next page">
	<div class="prev-next-info">
	<p class="prev-next-subtitle">next</p>
	<p class="prev-next-title">Upgrading PySpark</p>
	</div>
	<i class="fa-solid fa-angle-right"></i>
	</a>
	</div></div>

	</div>

	</footer>

	</div>



	<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">

	<div class="sidebar-secondary-item">
	<div class="tocsection sourcelink">
	<a href="../_sources/migration_guide/index.rst.txt">
	<i class="fa-solid fa-file-lines"></i> Show Source
	</a>
	</div>
	</div>

	</div></div>


	</div>
	<footer class="bd-footer-content">

	</footer>

	</main>
	</div>
	</div>

	<!-- Scripts loaded after <body> so the DOM is not blocked -->
	<script src="../_static/scripts/bootstrap.js?digest=e353d410970836974a52"></script>
	<script src="../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52"></script>

	<footer class="bd-footer">
	<div class="bd-footer__inner bd-page-width">

	<div class="footer-items__start">

	<div class="footer-item"><p class="copyright">
	Copyright @ 2025 The Apache Software Foundation, Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
	</p></div>

	<div class="footer-item">
	<p class="sphinx-version">
	Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 4.5.0.
	<br/>
	</p>
	</div>

	</div>


	<div class="footer-items__end">

	<div class="footer-item"><p class="theme-version">
	Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.13.3.
	</p></div>

	</div>

	</div>

	</footer>
	</body>
	</html>