site/docs/4.0.0-preview2/api/python/_modules/pyspark/logger/logger.html - spark-website - Git at Google



 <!DOCTYPE html>


 <html >

   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>pyspark.logger.logger &#8212; PySpark 4.0.0-preview2 documentation</title>


   <script data-cfasync="false">
     document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
     document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
   </script>

   <!-- Loaded before other Sphinx assets -->
   <link href="../../../_static/styles/theme.css?digest=e353d410970836974a52" rel="stylesheet" />
 <link href="../../../_static/styles/bootstrap.css?digest=e353d410970836974a52" rel="stylesheet" />
 <link href="../../../_static/styles/pydata-sphinx-theme.css?digest=e353d410970836974a52" rel="stylesheet" />


   <link href="../../../_static/vendor/fontawesome/6.1.2/css/all.min.css?digest=e353d410970836974a52" rel="stylesheet" />
   <link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.1.2/webfonts/fa-solid-900.woff2" />
 <link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.1.2/webfonts/fa-brands-400.woff2" />
 <link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.1.2/webfonts/fa-regular-400.woff2" />

     <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" />
     <link rel="stylesheet" type="text/css" href="../../../_static/css/pyspark.css" />

   <!-- Pre-loaded scripts that we'll load fully later -->
   <link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=e353d410970836974a52" />
 <link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52" />

     <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
     <script src="../../../_static/jquery.js"></script>
     <script src="../../../_static/underscore.js"></script>
     <script src="../../../_static/doctools.js"></script>
     <script src="../../../_static/clipboard.min.js"></script>
     <script src="../../../_static/copybutton.js"></script>
     <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
     <script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyspark/logger/logger';</script>
     <link rel="canonical" href="https://spark.apache.org/docs/latest/api/python/_modules/pyspark/logger/logger.html" />
     <link rel="search" title="Search" href="../../../search.html" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="docsearch:language" content="None">


     <!-- Matomo -->
     <script type="text/javascript">
         var _paq = window._paq = window._paq || [];
         /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
         _paq.push(["disableCookies"]);
         _paq.push(['trackPageView']);
         _paq.push(['enableLinkTracking']);
         (function() {
             var u="https://analytics.apache.org/";
             _paq.push(['setTrackerUrl', u+'matomo.php']);
             _paq.push(['setSiteId', '40']);
             var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
             g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
         })();
     </script>
     <!-- End Matomo Code -->

   </head>


   <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">


   <a class="skip-link" href="#main-content">Skip to main content</a>

   <input type="checkbox"
           class="sidebar-toggle"
           name="__primary"
           id="__primary"/>
   <label class="overlay overlay-primary" for="__primary"></label>

   <input type="checkbox"
           class="sidebar-toggle"
           name="__secondary"
           id="__secondary"/>
   <label class="overlay overlay-secondary" for="__secondary"></label>

   <div class="search-button__wrapper">
     <div class="search-button__overlay"></div>
     <div class="search-button__search-container">
 <form class="bd-search d-flex align-items-center"
       action="../../../search.html"
       method="get">
   <i class="fa-solid fa-magnifying-glass"></i>
   <input type="search"
          class="form-control"
          name="q"
          id="search-input"
          placeholder="Search the docs ..."
          aria-label="Search the docs ..."
          autocomplete="off"
          autocorrect="off"
          autocapitalize="off"
          spellcheck="false"/>
   <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
 </form></div>
   </div>

     <nav class="bd-header navbar navbar-expand-lg bd-navbar">
 <div class="bd-header__inner bd-page-width">
   <label class="sidebar-toggle primary-toggle" for="__primary">
     <span class="fa-solid fa-bars"></span>
   </label>

   <div class="navbar-header-items__start">

       <div class="navbar-item">


 <a class="navbar-brand logo" href="../../../index.html">


     <img src="https://spark.apache.org/images/spark-logo.png" class="logo__image only-light" alt="Logo image"/>
     <script>document.write(`<img src="https://spark.apache.org/images/spark-logo-rev.svg" class="logo__image only-dark" alt="Logo image"/>`);</script>


 </a></div>

   </div>


   <div class="col-lg-9 navbar-header-items">

     <div class="me-auto navbar-header-items__center">

         <div class="navbar-item"><nav class="navbar-nav">
   <p class="sidebar-header-items__title"
      role="heading"
      aria-level="1"
      aria-label="Site Navigation">
     Site Navigation
   </p>
   <ul class="bd-navbar-elements navbar-nav">

                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../index.html">
                         Overview
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../getting_started/index.html">
                         Getting Started
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../user_guide/index.html">
                         User Guides
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../reference/index.html">
                         API Reference
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../development/index.html">
                         Development
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../migration_guide/index.html">
                         Migration Guides
                       </a>
                     </li>

   </ul>
 </nav></div>

     </div>


     <div class="navbar-header-items__end">

         <div class="navbar-item navbar-persistent--container">

 <script>
 document.write(`
   <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <i class="fa-solid fa-magnifying-glass"></i>
   </button>
 `);
 </script>
         </div>


         <div class="navbar-item"><!--
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->

 <div id="version-button" class="dropdown">
     <button type="button" class="btn btn-secondary btn-sm navbar-btn dropdown-toggle" id="version_switcher_button" data-toggle="dropdown">
         4.0.0-preview2
         <span class="caret"></span>
     </button>
     <div id="version_switcher" class="dropdown-menu list-group-flush py-0" aria-labelledby="version_switcher_button">
     <!-- dropdown will be populated by javascript on page load -->
     </div>
 </div>

 <script type="text/javascript">
 // Function to construct the target URL from the JSON components
 function buildURL(entry) {
     var template = "https://spark.apache.org/docs/{version}/api/python/index.html";  // supplied by jinja
     template = template.replace("{version}", entry.version);
     return template;
 }

 // Function to check if corresponding page path exists in other version of docs
 // and, if so, go there instead of the homepage of the other docs version
 function checkPageExistsAndRedirect(event) {
     const currentFilePath = "_modules/pyspark/logger/logger.html",
           otherDocsHomepage = event.target.getAttribute("href");
     let tryUrl = `${otherDocsHomepage}${currentFilePath}`;
     $.ajax({
         type: 'HEAD',
         url: tryUrl,
         // if the page exists, go there
         success: function() {
             location.href = tryUrl;
         }
     }).fail(function() {
         location.href = otherDocsHomepage;
     });
     return false;
 }

 // Function to populate the version switcher
 (function () {
     // get JSON config
     $.getJSON("https://spark.apache.org/static/versions.json", function(data, textStatus, jqXHR) {
         // create the nodes first (before AJAX calls) to ensure the order is
         // correct (for now, links will go to doc version homepage)
         $.each(data, function(index, entry) {
             // if no custom name specified (e.g., "latest"), use version string
             if (!("name" in entry)) {
                 entry.name = entry.version;
             }
             // construct the appropriate URL, and add it to the dropdown
             entry.url = buildURL(entry);
             const node = document.createElement("a");
             node.setAttribute("class", "list-group-item list-group-item-action py-1");
             node.setAttribute("href", `${entry.url}`);
             node.textContent = `${entry.name}`;
             node.onclick = checkPageExistsAndRedirect;
             $("#version_switcher").append(node);
         });
     });
 })();
 </script></div>

         <div class="navbar-item">
 <script>
 document.write(`
   <button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
     <span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
     <span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
   </button>
 `);
 </script></div>

         <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
             <label class="sr-only">GitHub</label></a>
         </li>
         <li class="nav-item">


           <a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
             <label class="sr-only">PyPI</label></a>
         </li>
 </ul></div>

     </div>

   </div>


     <div class="navbar-persistent--mobile">
 <script>
 document.write(`
   <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <i class="fa-solid fa-magnifying-glass"></i>
   </button>
 `);
 </script>
     </div>


 </div>

     </nav>

   <div class="bd-container">
     <div class="bd-container__inner bd-page-width">

       <div class="bd-sidebar-primary bd-sidebar hide-on-wide">


   <div class="sidebar-header-items sidebar-primary__section">


       <div class="sidebar-header-items__center">

           <div class="navbar-item"><nav class="navbar-nav">
   <p class="sidebar-header-items__title"
      role="heading"
      aria-level="1"
      aria-label="Site Navigation">
     Site Navigation
   </p>
   <ul class="bd-navbar-elements navbar-nav">

                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../index.html">
                         Overview
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../getting_started/index.html">
                         Getting Started
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../user_guide/index.html">
                         User Guides
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../reference/index.html">
                         API Reference
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../development/index.html">
                         Development
                       </a>
                     </li>


                     <li class="nav-item">
                       <a class="nav-link nav-internal" href="../../../migration_guide/index.html">
                         Migration Guides
                       </a>
                     </li>

   </ul>
 </nav></div>

       </div>


       <div class="sidebar-header-items__end">

           <div class="navbar-item"><!--
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->

 <div id="version-button" class="dropdown">
     <button type="button" class="btn btn-secondary btn-sm navbar-btn dropdown-toggle" id="version_switcher_button" data-toggle="dropdown">
         4.0.0-preview2
         <span class="caret"></span>
     </button>
     <div id="version_switcher" class="dropdown-menu list-group-flush py-0" aria-labelledby="version_switcher_button">
     <!-- dropdown will be populated by javascript on page load -->
     </div>
 </div>

 <script type="text/javascript">
 // Function to construct the target URL from the JSON components
 function buildURL(entry) {
     var template = "https://spark.apache.org/docs/{version}/api/python/index.html";  // supplied by jinja
     template = template.replace("{version}", entry.version);
     return template;
 }

 // Function to check if corresponding page path exists in other version of docs
 // and, if so, go there instead of the homepage of the other docs version
 function checkPageExistsAndRedirect(event) {
     const currentFilePath = "_modules/pyspark/logger/logger.html",
           otherDocsHomepage = event.target.getAttribute("href");
     let tryUrl = `${otherDocsHomepage}${currentFilePath}`;
     $.ajax({
         type: 'HEAD',
         url: tryUrl,
         // if the page exists, go there
         success: function() {
             location.href = tryUrl;
         }
     }).fail(function() {
         location.href = otherDocsHomepage;
     });
     return false;
 }

 // Function to populate the version switcher
 (function () {
     // get JSON config
     $.getJSON("https://spark.apache.org/static/versions.json", function(data, textStatus, jqXHR) {
         // create the nodes first (before AJAX calls) to ensure the order is
         // correct (for now, links will go to doc version homepage)
         $.each(data, function(index, entry) {
             // if no custom name specified (e.g., "latest"), use version string
             if (!("name" in entry)) {
                 entry.name = entry.version;
             }
             // construct the appropriate URL, and add it to the dropdown
             entry.url = buildURL(entry);
             const node = document.createElement("a");
             node.setAttribute("class", "list-group-item list-group-item-action py-1");
             node.setAttribute("href", `${entry.url}`);
             node.textContent = `${entry.name}`;
             node.onclick = checkPageExistsAndRedirect;
             $("#version_switcher").append(node);
         });
     });
 })();
 </script></div>

           <div class="navbar-item">
 <script>
 document.write(`
   <button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
     <span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
     <span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
     <span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
   </button>
 `);
 </script></div>

           <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/spark" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-github"></i></span>
             <label class="sr-only">GitHub</label></a>
         </li>
         <li class="nav-item">


           <a href="https://pypi.org/project/pyspark" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-solid fa-box"></i></span>
             <label class="sr-only">PyPI</label></a>
         </li>
 </ul></div>

       </div>

   </div>


   <div class="sidebar-primary-items__end sidebar-primary__section">
   </div>

   <div id="rtd-footer-container"></div>


       </div>

       <main id="main-content" class="bd-main">


           <div class="bd-content">
             <div class="bd-article-container">

               <div class="bd-header-article">
 <div class="header-article-items header-article__inner">

     <div class="header-article-items__start">

         <div class="header-article-item">


 <nav aria-label="Breadcrumbs">
   <ul class="bd-breadcrumbs" role="navigation" aria-label="Breadcrumb">

     <li class="breadcrumb-item breadcrumb-home">
       <a href="../../../index.html" class="nav-link" aria-label="Home">
         <i class="fa-solid fa-home"></i>
       </a>
     </li>

     <li class="breadcrumb-item"><a href="../../index.html" class="nav-link">Module code</a></li>

     <li class="breadcrumb-item active" aria-current="page">pyspark.logger.logger</li>
   </ul>
 </nav>
 </div>

     </div>


 </div>
 </div>


 <div id="searchbox"></div>
                 <article class="bd-article" role="main">

   <h1>Source code for pyspark.logger.logger</h1><div class="highlight"><pre>
 <span></span><span class="c1"># -*- encoding: utf-8 -*-</span>
 <span class="c1">#</span>
 <span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
 <span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
 <span class="c1"># this work for additional information regarding copyright ownership.</span>
 <span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
 <span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
 <span class="c1"># the License.  You may obtain a copy of the License at</span>
 <span class="c1">#</span>
 <span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
 <span class="c1">#</span>
 <span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
 <span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 <span class="c1">#</span>

 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">cast</span><span class="p">,</span> <span class="n">Optional</span>


 <span class="k">class</span> <span class="nc">JSONFormatter</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Custom JSON formatter for logging records.</span>

 <span class="sd">    This formatter converts the log record to a JSON object with the following fields:</span>
 <span class="sd">    - timestamp: The time the log record was created.</span>
 <span class="sd">    - level: The log level of the record.</span>
 <span class="sd">    - name: The name of the logger.</span>
 <span class="sd">    - message: The log message.</span>
 <span class="sd">    - kwargs: Any additional keyword arguments passed to the logger.</span>
 <span class="sd">    &quot;&quot;&quot;</span>

     <span class="k">def</span> <span class="nf">format</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">record</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">LogRecord</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Format the specified record as a JSON string.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        record : logging.LogRecord</span>
 <span class="sd">            The log record to be formatted.</span>

 <span class="sd">        Returns</span>
 <span class="sd">        -------</span>
 <span class="sd">        str</span>
 <span class="sd">            The formatted log record as a JSON string.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">log_entry</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s2">&quot;ts&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">formatTime</span><span class="p">(</span><span class="n">record</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">datefmt</span><span class="p">),</span>
             <span class="s2">&quot;level&quot;</span><span class="p">:</span> <span class="n">record</span><span class="o">.</span><span class="n">levelname</span><span class="p">,</span>
             <span class="s2">&quot;logger&quot;</span><span class="p">:</span> <span class="n">record</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
             <span class="s2">&quot;msg&quot;</span><span class="p">:</span> <span class="n">record</span><span class="o">.</span><span class="n">getMessage</span><span class="p">(),</span>
             <span class="s2">&quot;context&quot;</span><span class="p">:</span> <span class="n">record</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;kwargs&quot;</span><span class="p">,</span> <span class="p">{}),</span>
         <span class="p">}</span>
         <span class="k">if</span> <span class="n">record</span><span class="o">.</span><span class="n">exc_info</span><span class="p">:</span>
             <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_value</span><span class="p">,</span> <span class="n">exc_tb</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">exc_info</span>
             <span class="n">log_entry</span><span class="p">[</span><span class="s2">&quot;exception&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
                 <span class="s2">&quot;class&quot;</span><span class="p">:</span> <span class="n">exc_type</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">if</span> <span class="n">exc_type</span> <span class="k">else</span> <span class="s2">&quot;UnknownException&quot;</span><span class="p">,</span>
                 <span class="s2">&quot;msg&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">(</span><span class="n">exc_value</span><span class="p">),</span>
                 <span class="s2">&quot;stacktrace&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">formatException</span><span class="p">(</span><span class="n">record</span><span class="o">.</span><span class="n">exc_info</span><span class="p">)</span><span class="o">.</span><span class="n">splitlines</span><span class="p">(),</span>
             <span class="p">}</span>
         <span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">log_entry</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>


 <div class="viewcode-block" id="PySparkLogger"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.html#pyspark.logger.PySparkLogger">[docs]</a><span class="k">class</span> <span class="nc">PySparkLogger</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">Logger</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Custom logging.Logger wrapper for PySpark that logs messages in a structured JSON format.</span>

 <span class="sd">    PySparkLogger extends the standard Python logging.Logger class, allowing seamless integration</span>
 <span class="sd">    with existing logging setups. It customizes the log output to JSON format, including additional</span>
 <span class="sd">    context information, making it more useful for PySpark applications.</span>

 <span class="sd">    .. versionadded:: 4.0.0</span>

 <span class="sd">    Example</span>
 <span class="sd">    -------</span>
 <span class="sd">    &gt;&gt;&gt; import logging</span>
 <span class="sd">    &gt;&gt;&gt; import json</span>
 <span class="sd">    &gt;&gt;&gt; from io import StringIO</span>
 <span class="sd">    &gt;&gt;&gt; from pyspark.logger import PySparkLogger</span>

 <span class="sd">    &gt;&gt;&gt; logger = PySparkLogger.getLogger(&quot;ExampleLogger&quot;)</span>
 <span class="sd">    &gt;&gt;&gt; logger.setLevel(logging.INFO)</span>
 <span class="sd">    &gt;&gt;&gt; stream = StringIO()</span>
 <span class="sd">    &gt;&gt;&gt; handler = logging.StreamHandler(stream)</span>
 <span class="sd">    &gt;&gt;&gt; logger.addHandler(handler)</span>

 <span class="sd">    &gt;&gt;&gt; logger.info(</span>
 <span class="sd">    ...     &quot;This is an informational message&quot;,</span>
 <span class="sd">    ...     extra={&quot;user&quot;: &quot;test_user&quot;, &quot;action&quot;: &quot;test_action&quot;}</span>
 <span class="sd">    ... )</span>
 <span class="sd">    &gt;&gt;&gt; log_output = stream.getvalue().strip().split(&#39;\\n&#39;)[0]</span>
 <span class="sd">    &gt;&gt;&gt; log = json.loads(log_output)</span>
 <span class="sd">    &gt;&gt;&gt; _ = log.pop(&quot;ts&quot;)  # Remove the timestamp field for static testing</span>

 <span class="sd">    &gt;&gt;&gt; print(json.dumps(log, ensure_ascii=False, indent=2))</span>
 <span class="sd">    {</span>
 <span class="sd">      &quot;level&quot;: &quot;INFO&quot;,</span>
 <span class="sd">      &quot;logger&quot;: &quot;ExampleLogger&quot;,</span>
 <span class="sd">      &quot;msg&quot;: &quot;This is an informational message&quot;,</span>
 <span class="sd">      &quot;context&quot;: {</span>
 <span class="sd">        &quot;extra&quot;: {</span>
 <span class="sd">          &quot;user&quot;: &quot;test_user&quot;,</span>
 <span class="sd">          &quot;action&quot;: &quot;test_action&quot;</span>
 <span class="sd">        }</span>
 <span class="sd">      }</span>
 <span class="sd">    }</span>
 <span class="sd">    &quot;&quot;&quot;</span>

     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;PySparkLogger&quot;</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">WARN</span><span class="p">)</span>
         <span class="n">_handler</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">_handler</span><span class="p">)</span>

 <div class="viewcode-block" id="PySparkLogger.addHandler"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.addHandler.html#pyspark.logger.PySparkLogger.addHandler">[docs]</a>    <span class="k">def</span> <span class="nf">addHandler</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">handler</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Handler</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Add the specified handler to this logger in structured JSON format.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">handler</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">JSONFormatter</span><span class="p">())</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">handler</span><span class="p">)</span></div>

 <div class="viewcode-block" id="PySparkLogger.getLogger"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.getLogger.html#pyspark.logger.PySparkLogger.getLogger">[docs]</a>    <span class="nd">@staticmethod</span>
     <span class="k">def</span> <span class="nf">getLogger</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;PySparkLogger&quot;</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Return a PySparkLogger with the specified name, creating it if necessary.</span>

 <span class="sd">        If no name is specified, return the logging.RootLogger.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        name : str, optional</span>
 <span class="sd">            The name of the logger.</span>

 <span class="sd">        Returns</span>
 <span class="sd">        -------</span>
 <span class="sd">        PySparkLogger</span>
 <span class="sd">            A configured instance of PySparkLogger.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">existing_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLoggerClass</span><span class="p">()</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">existing_logger</span><span class="p">,</span> <span class="n">PySparkLogger</span><span class="p">):</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">setLoggerClass</span><span class="p">(</span><span class="n">PySparkLogger</span><span class="p">)</span>

         <span class="n">pyspark_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
         <span class="c1"># Reset to the existing logger</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">setLoggerClass</span><span class="p">(</span><span class="n">existing_logger</span><span class="p">)</span>

         <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">PySparkLogger</span><span class="p">,</span> <span class="n">pyspark_logger</span><span class="p">)</span></div>

 <div class="viewcode-block" id="PySparkLogger.info"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.info.html#pyspark.logger.PySparkLogger.info">[docs]</a>    <span class="k">def</span> <span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">object</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Log &#39;msg % args&#39; with severity &#39;INFO&#39; in structured JSON format.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        msg : str</span>
 <span class="sd">            The log message.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;kwargs&quot;</span><span class="p">:</span> <span class="n">kwargs</span><span class="p">})</span></div>

 <div class="viewcode-block" id="PySparkLogger.warning"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.warning.html#pyspark.logger.PySparkLogger.warning">[docs]</a>    <span class="k">def</span> <span class="nf">warning</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">object</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Log &#39;msg % args&#39; with severity &#39;WARNING&#39; in structured JSON format.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        msg : str</span>
 <span class="sd">            The log message.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;kwargs&quot;</span><span class="p">:</span> <span class="n">kwargs</span><span class="p">})</span></div>

 <div class="viewcode-block" id="PySparkLogger.error"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.error.html#pyspark.logger.PySparkLogger.error">[docs]</a>    <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">object</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Log &#39;msg % args&#39; with severity &#39;ERROR&#39; in structured JSON format.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        msg : str</span>
 <span class="sd">            The log message.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;kwargs&quot;</span><span class="p">:</span> <span class="n">kwargs</span><span class="p">})</span></div>

 <div class="viewcode-block" id="PySparkLogger.exception"><a class="viewcode-back" href="../../../reference/api/pyspark.logger.PySparkLogger.exception.html#pyspark.logger.PySparkLogger.exception">[docs]</a>    <span class="k">def</span> <span class="nf">exception</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="nb">object</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">object</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Convenience method for logging an ERROR with exception information.</span>

 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
 <span class="sd">        msg : str</span>
 <span class="sd">            The log message.</span>
 <span class="sd">        exc_info : bool = True</span>
 <span class="sd">            If True, exception information is added to the logging message.</span>
 <span class="sd">            This includes the exception type, value, and traceback. Default is True.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">exc_info</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;kwargs&quot;</span><span class="p">:</span> <span class="n">kwargs</span><span class="p">})</span></div></div>
 </pre></div>

                 </article>


                 <footer class="bd-footer-article">

 <div class="footer-article-items footer-article__inner">

     <div class="footer-article-item"><!-- Previous / next buttons -->
 <div class="prev-next-area">
 </div></div>

 </div>

                 </footer>

             </div>


           </div>
           <footer class="bd-footer-content">

           </footer>

       </main>
     </div>
   </div>

   <!-- Scripts loaded after <body> so the DOM is not blocked -->
   <script src="../../../_static/scripts/bootstrap.js?digest=e353d410970836974a52"></script>
 <script src="../../../_static/scripts/pydata-sphinx-theme.js?digest=e353d410970836974a52"></script>

   <footer class="bd-footer">
 <div class="bd-footer__inner bd-page-width">

     <div class="footer-items__start">

         <div class="footer-item"><p class="copyright">
     Copyright @ 2024 The Apache Software Foundation, Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
 </p></div>

         <div class="footer-item">
   <p class="sphinx-version">
     Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 4.5.0.
     <br/>
   </p>
 </div>

     </div>


     <div class="footer-items__end">

         <div class="footer-item"><p class="theme-version">
   Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.13.3.
 </p></div>

     </div>

 </div>

   </footer>
   </body>
 </html>