| |
| <!DOCTYPE html> |
| <!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]--> |
| <!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]--> |
| <!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Data Types - Spark 3.5.0 Documentation</title> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="css/bootstrap.min.css"> |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet"> |
| <link href="css/custom.css" rel="stylesheet"> |
| <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script> |
| |
| <link rel="stylesheet" href="css/pygments-default.css"> |
| <link rel="stylesheet" href="css/docsearch.min.css" /> |
| <link rel="stylesheet" href="css/docsearch.css"> |
| |
| <!-- Matomo --> |
| <script type="text/javascript"> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '40']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| </head> |
| <body class="global"> |
| <!--[if lt IE 7]> |
| <p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p> |
| <![endif]--> |
| |
| <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html --> |
| |
| <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar"> |
| <div class="navbar-brand"><a href="index.html"> |
| <img src="img/spark-logo-rev.svg" width="141" height="72"/></a><span class="version">3.5.0</span> |
| </div> |
| <button class="navbar-toggler" type="button" data-toggle="collapse" |
| data-target="#navbarCollapse" aria-controls="navbarCollapse" |
| aria-expanded="false" aria-label="Toggle navigation"> |
| <span class="navbar-toggler-icon"></span> |
| </button> |
| <div class="collapse navbar-collapse" id="navbarCollapse"> |
| <ul class="navbar-nav me-auto"> |
| <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a> |
| <div class="dropdown-menu" aria-labelledby="navbarQuickStart"> |
| <a class="dropdown-item" href="quick-start.html">Quick Start</a> |
| <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a> |
| <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a> |
| <a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a> |
| <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a> |
| <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a> |
| <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a> |
| <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a> |
| <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a> |
| <div class="dropdown-menu" aria-labelledby="navbarAPIDocs"> |
| <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a> |
| <a class="dropdown-item" href="api/java/index.html">Java</a> |
| <a class="dropdown-item" href="api/python/index.html">Python</a> |
| <a class="dropdown-item" href="api/R/index.html">R</a> |
| <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a> |
| <div class="dropdown-menu" aria-labelledby="navbarDeploying"> |
| <a class="dropdown-item" href="cluster-overview.html">Overview</a> |
| <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a> |
| <div class="dropdown-divider"></div> |
| <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a> |
| <a class="dropdown-item" href="running-on-mesos.html">Mesos</a> |
| <a class="dropdown-item" href="running-on-yarn.html">YARN</a> |
| <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a> |
| </div> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a> |
| <div class="dropdown-menu" aria-labelledby="navbarMore"> |
| <a class="dropdown-item" href="configuration.html">Configuration</a> |
| <a class="dropdown-item" href="monitoring.html">Monitoring</a> |
| <a class="dropdown-item" href="tuning.html">Tuning Guide</a> |
| <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a> |
| <a class="dropdown-item" href="security.html">Security</a> |
| <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a> |
| <a class="dropdown-item" href="migration-guide.html">Migration Guide</a> |
| <div class="dropdown-divider"></div> |
| <a class="dropdown-item" href="building-spark.html">Building Spark</a> |
| <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a> |
| <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a> |
| </div> |
| </li> |
| |
| <li class="nav-item"> |
| <input type="text" id="docsearch-input" placeholder="Search the docs…"> |
| </li> |
| </ul> |
| <!--<span class="navbar-text navbar-right"><span class="version-text">v3.5.0</span></span>--> |
| </div> |
| </nav> |
| |
| |
| |
| <div class="container"> |
| |
| |
| |
| <div class="left-menu-wrapper"> |
| <div class="left-menu"> |
| <h3><a href="sql-programming-guide.html">Spark SQL Guide</a></h3> |
| |
| <ul> |
| |
| <li> |
| <a href="sql-getting-started.html"> |
| |
| Getting Started |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-data-sources.html"> |
| |
| Data Sources |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-performance-tuning.html"> |
| |
| Performance Tuning |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-distributed-sql-engine.html"> |
| |
| Distributed SQL Engine |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-pyspark-pandas-with-arrow.html"> |
| |
| PySpark Usage Guide for Pandas with Apache Arrow |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-migration-guide.html"> |
| |
| Migration Guide |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref.html"> |
| |
| SQL Reference |
| |
| </a> |
| </li> |
| |
| |
| |
| <ul> |
| |
| <li> |
| <a href="sql-ref-ansi-compliance.html"> |
| |
| ANSI Compliance |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-datatypes.html"> |
| |
| Data Types |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-datetime-pattern.html"> |
| |
| Datetime Pattern |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-number-pattern.html"> |
| |
| Number Pattern |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-functions.html"> |
| |
| Functions |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-identifier.html"> |
| |
| Identifiers |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-literals.html"> |
| |
| Literals |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-null-semantics.html"> |
| |
| Null Semantics |
| |
| </a> |
| </li> |
| |
| |
| |
| <li> |
| <a href="sql-ref-syntax.html"> |
| |
| SQL Syntax |
| |
| </a> |
| </li> |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li> |
| <a href="sql-error-conditions.html"> |
| |
| Error Conditions |
| |
| </a> |
| </li> |
| |
| |
| |
| </ul> |
| |
| </div> |
| </div> |
| |
| <input id="nav-trigger" class="nav-trigger" checked type="checkbox"> |
| <label for="nav-trigger"></label> |
| <div class="content-with-sidebar mr-3" id="content"> |
| |
| <h1 class="title">Data Types</h1> |
| |
| |
| <h3 id="supported-data-types">Supported Data Types</h3> |
| |
| <p>Spark SQL and DataFrames support the following data types:</p> |
| |
| <ul> |
| <li>Numeric types |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">ByteType</code>: Represents 1-byte signed integer numbers. |
| The range of numbers is from <code class="language-plaintext highlighter-rouge">-128</code> to <code class="language-plaintext highlighter-rouge">127</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">ShortType</code>: Represents 2-byte signed integer numbers. |
| The range of numbers is from <code class="language-plaintext highlighter-rouge">-32768</code> to <code class="language-plaintext highlighter-rouge">32767</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">IntegerType</code>: Represents 4-byte signed integer numbers. |
| The range of numbers is from <code class="language-plaintext highlighter-rouge">-2147483648</code> to <code class="language-plaintext highlighter-rouge">2147483647</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">LongType</code>: Represents 8-byte signed integer numbers. |
| The range of numbers is from <code class="language-plaintext highlighter-rouge">-9223372036854775808</code> to <code class="language-plaintext highlighter-rouge">9223372036854775807</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">FloatType</code>: Represents 4-byte single-precision floating point numbers.</li> |
| <li><code class="language-plaintext highlighter-rouge">DoubleType</code>: Represents 8-byte double-precision floating point numbers.</li> |
| <li><code class="language-plaintext highlighter-rouge">DecimalType</code>: Represents arbitrary-precision signed decimal numbers. Backed internally by <code class="language-plaintext highlighter-rouge">java.math.BigDecimal</code>. A <code class="language-plaintext highlighter-rouge">BigDecimal</code> consists of an arbitrary precision integer unscaled value and a 32-bit integer scale.</li> |
| </ul> |
| </li> |
| <li>String type |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">StringType</code>: Represents character string values.</li> |
| <li><code class="language-plaintext highlighter-rouge">VarcharType(length)</code>: A variant of <code class="language-plaintext highlighter-rouge">StringType</code> which has a length limitation. Data writing will fail if the input string exceeds the length limitation. Note: this type can only be used in table schema, not functions/operators.</li> |
| <li><code class="language-plaintext highlighter-rouge">CharType(length)</code>: A variant of <code class="language-plaintext highlighter-rouge">VarcharType(length)</code> which is fixed length. Reading column of type <code class="language-plaintext highlighter-rouge">CharType(n)</code> always returns string values of length <code class="language-plaintext highlighter-rouge">n</code>. Char type column comparison will pad the short one to the longer length.</li> |
| </ul> |
| </li> |
| <li>Binary type |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">BinaryType</code>: Represents byte sequence values.</li> |
| </ul> |
| </li> |
| <li>Boolean type |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">BooleanType</code>: Represents boolean values.</li> |
| </ul> |
| </li> |
| <li>Datetime type |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">DateType</code>: Represents values comprising values of fields year, month and day, without a |
| time-zone.</li> |
| <li><code class="language-plaintext highlighter-rouge">TimestampType</code>: Timestamp with local time zone(TIMESTAMP_LTZ). It represents values comprising values of fields year, month, day, |
| hour, minute, and second, with the session local time-zone. The timestamp value represents an |
| absolute point in time.</li> |
| <li><code class="language-plaintext highlighter-rouge">TimestampNTZType</code>: Timestamp without time zone(TIMESTAMP_NTZ). It represents values comprising values of fields year, month, day, |
| hour, minute, and second. All operations are performed without taking any time zone into account. |
| <ul> |
| <li>Note: TIMESTAMP in Spark is a user-specified alias associated with one of the TIMESTAMP_LTZ and TIMESTAMP_NTZ variations. Users can set the default timestamp type as <code class="language-plaintext highlighter-rouge">TIMESTAMP_LTZ</code>(default value) or <code class="language-plaintext highlighter-rouge">TIMESTAMP_NTZ</code> via the configuration <code class="language-plaintext highlighter-rouge">spark.sql.timestampType</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>Interval types |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">YearMonthIntervalType(startField, endField)</code>: Represents a year-month interval which is made up of a contiguous subset of the following fields: |
| <ul> |
| <li>MONTH, months within years <code class="language-plaintext highlighter-rouge">[0..11]</code>,</li> |
| <li>YEAR, years in the range <code class="language-plaintext highlighter-rouge">[0..178956970]</code>.</li> |
| </ul> |
| |
| <p>Individual interval fields are non-negative, but an interval itself can have a sign, and be negative.</p> |
| |
| <p><code class="language-plaintext highlighter-rouge">startField</code> is the leftmost field, and <code class="language-plaintext highlighter-rouge">endField</code> is the rightmost field of the type. Valid values of <code class="language-plaintext highlighter-rouge">startField</code> and <code class="language-plaintext highlighter-rouge">endField</code> are 0(MONTH) and 1(YEAR). Supported year-month interval types are:</p> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Year-Month Interval Type</th> |
| <th>SQL type</th> |
| <th>An instance of the type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">YearMonthIntervalType(YEAR, YEAR)</code> or <code class="language-plaintext highlighter-rouge">YearMonthIntervalType(YEAR)</code></td> |
| <td>INTERVAL YEAR</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '2021' YEAR</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">YearMonthIntervalType(YEAR, MONTH)</code></td> |
| <td>INTERVAL YEAR TO MONTH</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '2021-07' YEAR TO MONTH</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">YearMonthIntervalType(MONTH, MONTH)</code> or <code class="language-plaintext highlighter-rouge">YearMonthIntervalType(MONTH)</code></td> |
| <td>INTERVAL MONTH</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '10' MONTH</code></td> |
| </tr> |
| </tbody> |
| </table> |
| </li> |
| <li><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(startField, endField)</code>: Represents a day-time interval which is made up of a contiguous subset of the following fields: |
| <ul> |
| <li>SECOND, seconds within minutes and possibly fractions of a second <code class="language-plaintext highlighter-rouge">[0..59.999999]</code>,</li> |
| <li>MINUTE, minutes within hours <code class="language-plaintext highlighter-rouge">[0..59]</code>,</li> |
| <li>HOUR, hours within days <code class="language-plaintext highlighter-rouge">[0..23]</code>,</li> |
| <li>DAY, days in the range <code class="language-plaintext highlighter-rouge">[0..106751991]</code>.</li> |
| </ul> |
| |
| <p>Individual interval fields are non-negative, but an interval itself can have a sign, and be negative.</p> |
| |
| <p><code class="language-plaintext highlighter-rouge">startField</code> is the leftmost field, and <code class="language-plaintext highlighter-rouge">endField</code> is the rightmost field of the type. Valid values of <code class="language-plaintext highlighter-rouge">startField</code> and <code class="language-plaintext highlighter-rouge">endField</code> are 0 (DAY), 1 (HOUR), 2 (MINUTE), 3 (SECOND). Supported day-time interval types are:</p> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Day-Time Interval Type</th> |
| <th>SQL type</th> |
| <th>An instance of the type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(DAY, DAY)</code> or <code class="language-plaintext highlighter-rouge">DayTimeIntervalType(DAY)</code></td> |
| <td>INTERVAL DAY</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '100' DAY</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(DAY, HOUR)</code></td> |
| <td>INTERVAL DAY TO HOUR</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '100 10' DAY TO HOUR</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(DAY, MINUTE)</code></td> |
| <td>INTERVAL DAY TO MINUTE</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '100 10:30' DAY TO MINUTE</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(DAY, SECOND)</code></td> |
| <td>INTERVAL DAY TO SECOND</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '100 10:30:40.999999' DAY TO SECOND</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(HOUR, HOUR)</code> or <code class="language-plaintext highlighter-rouge">DayTimeIntervalType(HOUR)</code></td> |
| <td>INTERVAL HOUR</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '123' HOUR</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(HOUR, MINUTE)</code></td> |
| <td>INTERVAL HOUR TO MINUTE</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '123:10' HOUR TO MINUTE</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(HOUR, SECOND)</code></td> |
| <td>INTERVAL HOUR TO SECOND</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '123:10:59' HOUR TO SECOND</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(MINUTE, MINUTE)</code> or <code class="language-plaintext highlighter-rouge">DayTimeIntervalType(MINUTE)</code></td> |
| <td>INTERVAL MINUTE</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '1000' MINUTE</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(MINUTE, SECOND)</code></td> |
| <td>INTERVAL MINUTE TO SECOND</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '1000:01.001' MINUTE TO SECOND</code></td> |
| </tr> |
| <tr> |
| <td><code class="language-plaintext highlighter-rouge">DayTimeIntervalType(SECOND, SECOND)</code> or <code class="language-plaintext highlighter-rouge">DayTimeIntervalType(SECOND)</code></td> |
| <td>INTERVAL SECOND</td> |
| <td><code class="language-plaintext highlighter-rouge">INTERVAL '1000.000001' SECOND</code></td> |
| </tr> |
| </tbody> |
| </table> |
| </li> |
| </ul> |
| </li> |
| <li>Complex types |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">ArrayType(elementType, containsNull)</code>: Represents values comprising a sequence of |
| elements with the type of <code class="language-plaintext highlighter-rouge">elementType</code>. <code class="language-plaintext highlighter-rouge">containsNull</code> is used to indicate if |
| elements in a <code class="language-plaintext highlighter-rouge">ArrayType</code> value can have <code class="language-plaintext highlighter-rouge">null</code> values.</li> |
| <li><code class="language-plaintext highlighter-rouge">MapType(keyType, valueType, valueContainsNull)</code>: |
| Represents values comprising a set of key-value pairs. The data type of keys is |
| described by <code class="language-plaintext highlighter-rouge">keyType</code> and the data type of values is described by <code class="language-plaintext highlighter-rouge">valueType</code>. |
| For a <code class="language-plaintext highlighter-rouge">MapType</code> value, keys are not allowed to have <code class="language-plaintext highlighter-rouge">null</code> values. <code class="language-plaintext highlighter-rouge">valueContainsNull</code> |
| is used to indicate if values of a <code class="language-plaintext highlighter-rouge">MapType</code> value can have <code class="language-plaintext highlighter-rouge">null</code> values.</li> |
| <li><code class="language-plaintext highlighter-rouge">StructType(fields)</code>: Represents values with the structure described by |
| a sequence of <code class="language-plaintext highlighter-rouge">StructField</code>s (<code class="language-plaintext highlighter-rouge">fields</code>). |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">StructField(name, dataType, nullable)</code>: Represents a field in a <code class="language-plaintext highlighter-rouge">StructType</code>. |
| The name of a field is indicated by <code class="language-plaintext highlighter-rouge">name</code>. The data type of a field is indicated |
| by <code class="language-plaintext highlighter-rouge">dataType</code>. <code class="language-plaintext highlighter-rouge">nullable</code> is used to indicate if values of these fields can have |
| <code class="language-plaintext highlighter-rouge">null</code> values.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| |
| <div class="codetabs"> |
| |
| <div data-lang="python"> |
| |
| <p>All data types of Spark SQL are located in the package of <code class="language-plaintext highlighter-rouge">pyspark.sql.types</code>. |
| You can access them by doing</p> |
| |
| <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="kn">import</span> <span class="o">*</span></code></pre></figure> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Data type</th> |
| <th>Value type in Python</th> |
| <th>API to access or create a data type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><strong>ByteType</strong></td> |
| <td>int or long<br /><strong>Note:</strong> Numbers will be converted to 1-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -128 to 127.</td> |
| <td>ByteType()</td> |
| </tr> |
| <tr> |
| <td><strong>ShortType</strong></td> |
| <td>int or long<br /><strong>Note:</strong> Numbers will be converted to 2-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -32768 to 32767.</td> |
| <td>ShortType()</td> |
| </tr> |
| <tr> |
| <td><strong>IntegerType</strong></td> |
| <td>int or long</td> |
| <td>IntegerType()</td> |
| </tr> |
| <tr> |
| <td><strong>LongType</strong></td> |
| <td>long<br /><strong>Note:</strong> Numbers will be converted to 8-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807. Otherwise, please convert data to decimal.Decimal and use DecimalType.</td> |
| <td>LongType()</td> |
| </tr> |
| <tr> |
| <td><strong>FloatType</strong></td> |
| <td>float<br /><strong>Note:</strong> Numbers will be converted to 4-byte single-precision floating point numbers at runtime.</td> |
| <td>FloatType()</td> |
| </tr> |
| <tr> |
| <td><strong>DoubleType</strong></td> |
| <td>float</td> |
| <td>DoubleType()</td> |
| </tr> |
| <tr> |
| <td><strong>DecimalType</strong></td> |
| <td>decimal.Decimal</td> |
| <td>DecimalType()</td> |
| </tr> |
| <tr> |
| <td><strong>StringType</strong></td> |
| <td>string</td> |
| <td>StringType()</td> |
| </tr> |
| <tr> |
| <td><strong>BinaryType</strong></td> |
| <td>bytearray</td> |
| <td>BinaryType()</td> |
| </tr> |
| <tr> |
| <td><strong>BooleanType</strong></td> |
| <td>bool</td> |
| <td>BooleanType()</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampType</strong></td> |
| <td>datetime.datetime</td> |
| <td>TimestampType()</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampNTZType</strong></td> |
| <td>datetime.datetime</td> |
| <td>TimestampNTZType()</td> |
| </tr> |
| <tr> |
| <td><strong>DateType</strong></td> |
| <td>datetime.date</td> |
| <td>DateType()</td> |
| </tr> |
| <tr> |
| <td><strong>DayTimeIntervalType</strong></td> |
| <td>datetime.timedelta</td> |
| <td>DayTimeIntervalType()</td> |
| </tr> |
| <tr> |
| <td><strong>ArrayType</strong></td> |
| <td>list, tuple, or array</td> |
| <td>ArrayType(<em>elementType</em>, [<em>containsNull</em>])<br /><strong>Note:</strong>The default value of <em>containsNull</em> is True.</td> |
| </tr> |
| <tr> |
| <td><strong>MapType</strong></td> |
| <td>dict</td> |
| <td>MapType(<em>keyType</em>, <em>valueType</em>, [<em>valueContainsNull]</em>)<br /><strong>Note:</strong>The default value of <em>valueContainsNull</em> is True.</td> |
| </tr> |
| <tr> |
| <td><strong>StructType</strong></td> |
| <td>list or tuple</td> |
| <td>StructType(<em>fields</em>)<br /><strong>Note:</strong> <em>fields</em> is a Seq of StructFields. Also, two fields with the same name are not allowed.</td> |
| </tr> |
| <tr> |
| <td><strong>StructField</strong></td> |
| <td>The value type in Python of the data type of this field<br />(For example, Int for a StructField with the data type IntegerType)</td> |
| <td>StructField(<em>name</em>, <em>dataType</em>, [<em>nullable</em>])<br /><strong>Note:</strong> The default value of <em>nullable</em> is True.</td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| <div data-lang="scala"> |
| |
| <p>All data types of Spark SQL are located in the package <code class="language-plaintext highlighter-rouge">org.apache.spark.sql.types</code>. |
| You can access them by doing</p> |
| |
| <div class="highlight"><pre class="codehilite"><code><span class="k">import</span> <span class="nn">org.apache.spark.sql.types._</span></code></pre></div> |
| <div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala" in the Spark repo.</small></div> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Data type</th> |
| <th>Value type in Scala</th> |
| <th>API to access or create a data type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><strong>ByteType</strong></td> |
| <td>Byte</td> |
| <td>ByteType</td> |
| </tr> |
| <tr> |
| <td><strong>ShortType</strong></td> |
| <td>Short</td> |
| <td>ShortType</td> |
| </tr> |
| <tr> |
| <td><strong>IntegerType</strong></td> |
| <td>Int</td> |
| <td>IntegerType</td> |
| </tr> |
| <tr> |
| <td><strong>LongType</strong></td> |
| <td>Long</td> |
| <td>LongType</td> |
| </tr> |
| <tr> |
| <td><strong>FloatType</strong></td> |
| <td>Float</td> |
| <td>FloatType</td> |
| </tr> |
| <tr> |
| <td><strong>DoubleType</strong></td> |
| <td>Double</td> |
| <td>DoubleType</td> |
| </tr> |
| <tr> |
| <td><strong>DecimalType</strong></td> |
| <td>java.math.BigDecimal</td> |
| <td>DecimalType</td> |
| </tr> |
| <tr> |
| <td><strong>StringType</strong></td> |
| <td>String</td> |
| <td>StringType</td> |
| </tr> |
| <tr> |
| <td><strong>BinaryType</strong></td> |
| <td>Array[Byte]</td> |
| <td>BinaryType</td> |
| </tr> |
| <tr> |
| <td><strong>BooleanType</strong></td> |
| <td>Boolean</td> |
| <td>BooleanType</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampType</strong></td> |
| <td>java.time.Instant or java.sql.Timestamp</td> |
| <td>TimestampType</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampNTZType</strong></td> |
| <td>java.time.LocalDateTime</td> |
| <td>TimestampNTZType</td> |
| </tr> |
| <tr> |
| <td><strong>DateType</strong></td> |
| <td>java.time.LocalDate or java.sql.Date</td> |
| <td>DateType</td> |
| </tr> |
| <tr> |
| <td><strong>YearMonthIntervalType</strong></td> |
| <td>java.time.Period</td> |
| <td>YearMonthIntervalType</td> |
| </tr> |
| <tr> |
| <td><strong>DayTimeIntervalType</strong></td> |
| <td>java.time.Duration</td> |
| <td>DayTimeIntervalType</td> |
| </tr> |
| <tr> |
| <td><strong>ArrayType</strong></td> |
| <td>scala.collection.Seq</td> |
| <td>ArrayType(<em>elementType</em>, [<em>containsNull]</em>)<br /><strong>Note:</strong> The default value of <em>containsNull</em> is true.</td> |
| </tr> |
| <tr> |
| <td><strong>MapType</strong></td> |
| <td>scala.collection.Map</td> |
| <td>MapType(<em>keyType</em>, <em>valueType</em>, [<em>valueContainsNull]</em>)<br /><strong>Note:</strong> The default value of <em>valueContainsNull</em> is true.</td> |
| </tr> |
| <tr> |
| <td><strong>StructType</strong></td> |
| <td>org.apache.spark.sql.Row</td> |
| <td>StructType(<em>fields</em>)<br /><strong>Note:</strong> <em>fields</em> is a Seq of StructFields. Also, two fields with the same name are not allowed.</td> |
| </tr> |
| <tr> |
| <td><strong>StructField</strong></td> |
| <td>The value type in Scala of the data type of this field(For example, Int for a StructField with the data type IntegerType)</td> |
| <td>StructField(<em>name</em>, <em>dataType</em>, [<em>nullable</em>])<br /><strong>Note:</strong> The default value of <em>nullable</em> is true.</td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| <div data-lang="java"> |
| |
| <p>All data types of Spark SQL are located in the package of |
| <code class="language-plaintext highlighter-rouge">org.apache.spark.sql.types</code>. To access or create a data type, |
| please use factory methods provided in |
| <code class="language-plaintext highlighter-rouge">org.apache.spark.sql.types.DataTypes</code>.</p> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Data type</th> |
| <th>Value type in Java</th> |
| <th>API to access or create a data type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><strong>ByteType</strong></td> |
| <td>byte or Byte</td> |
| <td>DataTypes.ByteType</td> |
| </tr> |
| <tr> |
| <td><strong>ShortType</strong></td> |
| <td>short or Short</td> |
| <td>DataTypes.ShortType</td> |
| </tr> |
| <tr> |
| <td><strong>IntegerType</strong></td> |
| <td>int or Integer</td> |
| <td>DataTypes.IntegerType</td> |
| </tr> |
| <tr> |
| <td><strong>LongType</strong></td> |
| <td>long or Long</td> |
| <td>DataTypes.LongType</td> |
| </tr> |
| <tr> |
| <td><strong>FloatType</strong></td> |
| <td>float or Float</td> |
| <td>DataTypes.FloatType</td> |
| </tr> |
| <tr> |
| <td><strong>DoubleType</strong></td> |
| <td>double or Double</td> |
| <td>DataTypes.DoubleType</td> |
| </tr> |
| <tr> |
| <td><strong>DecimalType</strong></td> |
| <td>java.math.BigDecimal</td> |
| <td>DataTypes.createDecimalType()<br />DataTypes.createDecimalType(<em>precision</em>, <em>scale</em>).</td> |
| </tr> |
| <tr> |
| <td><strong>StringType</strong></td> |
| <td>String</td> |
| <td>DataTypes.StringType</td> |
| </tr> |
| <tr> |
| <td><strong>BinaryType</strong></td> |
| <td>byte[]</td> |
| <td>DataTypes.BinaryType</td> |
| </tr> |
| <tr> |
| <td><strong>BooleanType</strong></td> |
| <td>boolean or Boolean</td> |
| <td>DataTypes.BooleanType</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampType</strong></td> |
| <td>java.time.Instant or java.sql.Timestamp</td> |
| <td>DataTypes.TimestampType</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampNTZType</strong></td> |
| <td>java.time.LocalDateTime</td> |
| <td>DataTypes.TimestampNTZType</td> |
| </tr> |
| <tr> |
| <td><strong>DateType</strong></td> |
| <td>java.time.LocalDate or java.sql.Date</td> |
| <td>DataTypes.DateType</td> |
| </tr> |
| <tr> |
| <td><strong>YearMonthIntervalType</strong></td> |
| <td>java.time.Period</td> |
| <td>DataTypes.YearMonthIntervalType</td> |
| </tr> |
| <tr> |
| <td><strong>DayTimeIntervalType</strong></td> |
| <td>java.time.Duration</td> |
| <td>DataTypes.DayTimeIntervalType</td> |
| </tr> |
| <tr> |
| <td><strong>ArrayType</strong></td> |
| <td>java.util.List</td> |
| <td>DataTypes.createArrayType(<em>elementType</em>)<br /><strong>Note:</strong> The value of <em>containsNull</em> will be true.<br />DataTypes.createArrayType(<em>elementType</em>, <em>containsNull</em>).</td> |
| </tr> |
| <tr> |
| <td><strong>MapType</strong></td> |
| <td>java.util.Map</td> |
| <td>DataTypes.createMapType(<em>keyType</em>, <em>valueType</em>)<br /><strong>Note:</strong> The value of <em>valueContainsNull</em> will be true.<br />DataTypes.createMapType(<em>keyType</em>, <em>valueType</em>, <em>valueContainsNull</em>)</td> |
| </tr> |
| <tr> |
| <td><strong>StructType</strong></td> |
| <td>org.apache.spark.sql.Row</td> |
| <td>DataTypes.createStructType(<em>fields</em>)<br /><strong>Note:</strong> <em>fields</em> is a List or an array of StructFields.Also, two fields with the same name are not allowed.</td> |
| </tr> |
| <tr> |
| <td><strong>StructField</strong></td> |
| <td>The value type in Java of the data type of this field (For example, int for a StructField with the data type IntegerType)</td> |
| <td>DataTypes.createStructField(<em>name</em>, <em>dataType</em>, <em>nullable</em>)</td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| <div data-lang="r"> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Data type</th> |
| <th>Value type in R</th> |
| <th>API to access or create a data type</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><strong>ByteType</strong></td> |
| <td>integer <br /><strong>Note:</strong> Numbers will be converted to 1-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -128 to 127.</td> |
| <td>“byte”</td> |
| </tr> |
| <tr> |
| <td><strong>ShortType</strong></td> |
| <td>integer <br /><strong>Note:</strong> Numbers will be converted to 2-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -32768 to 32767.</td> |
| <td>“short”</td> |
| </tr> |
| <tr> |
| <td><strong>IntegerType</strong></td> |
| <td>integer</td> |
| <td>“integer”</td> |
| </tr> |
| <tr> |
| <td><strong>LongType</strong></td> |
| <td>integer <br /><strong>Note:</strong> Numbers will be converted to 8-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807. Otherwise, please convert data to decimal.Decimal and use DecimalType.</td> |
| <td>“long”</td> |
| </tr> |
| <tr> |
| <td><strong>FloatType</strong></td> |
| <td>numeric <br /><strong>Note:</strong> Numbers will be converted to 4-byte single-precision floating point numbers at runtime.</td> |
| <td>“float”</td> |
| </tr> |
| <tr> |
| <td><strong>DoubleType</strong></td> |
| <td>numeric</td> |
| <td>“double”</td> |
| </tr> |
| <tr> |
| <td><strong>DecimalType</strong></td> |
| <td>Not supported</td> |
| <td>Not supported</td> |
| </tr> |
| <tr> |
| <td><strong>StringType</strong></td> |
| <td>character</td> |
| <td>“string”</td> |
| </tr> |
| <tr> |
| <td><strong>BinaryType</strong></td> |
| <td>raw</td> |
| <td>“binary”</td> |
| </tr> |
| <tr> |
| <td><strong>BooleanType</strong></td> |
| <td>logical</td> |
| <td>“bool”</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampType</strong></td> |
| <td>POSIXct</td> |
| <td>“timestamp”</td> |
| </tr> |
| <tr> |
| <td><strong>DateType</strong></td> |
| <td>Date</td> |
| <td>“date”</td> |
| </tr> |
| <tr> |
| <td><strong>ArrayType</strong></td> |
| <td>vector or list</td> |
| <td>list(type=”array”, elementType=<em>elementType</em>, containsNull=[<em>containsNull</em>])<br /><strong>Note:</strong> The default value of <em>containsNull</em> is TRUE.</td> |
| </tr> |
| <tr> |
| <td><strong>MapType</strong></td> |
| <td>environment</td> |
| <td>list(type=”map”, keyType=<em>keyType</em>, valueType=<em>valueType</em>, valueContainsNull=[<em>valueContainsNull</em>])<br /> <strong>Note:</strong> The default value of <em>valueContainsNull</em> is TRUE.</td> |
| </tr> |
| <tr> |
| <td><strong>StructType</strong></td> |
| <td>named list</td> |
| <td>list(type=”struct”, fields=<em>fields</em>)<br /> <strong>Note:</strong> <em>fields</em> is a Seq of StructFields. Also, two fields with the same name are not allowed.</td> |
| </tr> |
| <tr> |
| <td><strong>StructField</strong></td> |
| <td>The value type in R of the data type of this field (For example, integer for a StructField with the data type IntegerType)</td> |
| <td>list(name=<em>name</em>, type=<em>dataType</em>, nullable=[<em>nullable</em>])<br /> <strong>Note:</strong> The default value of <em>nullable</em> is TRUE.</td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| <div data-lang="SQL"> |
| |
| <p>The following table shows the type names as well as aliases used in Spark SQL parser for each data type.</p> |
| |
| <table> |
| <thead> |
| <tr> |
| <th>Data type</th> |
| <th>SQL name</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td><strong>BooleanType</strong></td> |
| <td>BOOLEAN</td> |
| </tr> |
| <tr> |
| <td><strong>ByteType</strong></td> |
| <td>BYTE, TINYINT</td> |
| </tr> |
| <tr> |
| <td><strong>ShortType</strong></td> |
| <td>SHORT, SMALLINT</td> |
| </tr> |
| <tr> |
| <td><strong>IntegerType</strong></td> |
| <td>INT, INTEGER</td> |
| </tr> |
| <tr> |
| <td><strong>LongType</strong></td> |
| <td>LONG, BIGINT</td> |
| </tr> |
| <tr> |
| <td><strong>FloatType</strong></td> |
| <td>FLOAT, REAL</td> |
| </tr> |
| <tr> |
| <td><strong>DoubleType</strong></td> |
| <td>DOUBLE</td> |
| </tr> |
| <tr> |
| <td><strong>DateType</strong></td> |
| <td>DATE</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampType</strong></td> |
| <td>TIMESTAMP, TIMESTAMP_LTZ</td> |
| </tr> |
| <tr> |
| <td><strong>TimestampNTZType</strong></td> |
| <td>TIMESTAMP_NTZ</td> |
| </tr> |
| <tr> |
| <td><strong>StringType</strong></td> |
| <td>STRING</td> |
| </tr> |
| <tr> |
| <td><strong>BinaryType</strong></td> |
| <td>BINARY</td> |
| </tr> |
| <tr> |
| <td><strong>DecimalType</strong></td> |
| <td>DECIMAL, DEC, NUMERIC</td> |
| </tr> |
| <tr> |
| <td><strong>YearMonthIntervalType</strong></td> |
| <td>INTERVAL YEAR, INTERVAL YEAR TO MONTH, INTERVAL MONTH</td> |
| </tr> |
| <tr> |
| <td><strong>DayTimeIntervalType</strong></td> |
| <td>INTERVAL DAY, INTERVAL DAY TO HOUR, INTERVAL DAY TO MINUTE, INTERVAL DAY TO SECOND, INTERVAL HOUR, INTERVAL HOUR TO MINUTE, INTERVAL HOUR TO SECOND, INTERVAL MINUTE, INTERVAL MINUTE TO SECOND, INTERVAL SECOND</td> |
| </tr> |
| <tr> |
| <td><strong>ArrayType</strong></td> |
| <td>ARRAY<element_type></td> |
| </tr> |
| <tr> |
| <td><strong>StructType</strong></td> |
| <td>STRUCT<field1_name: field1_type, field2_name: field2_type, …><br /> <strong>Note:</strong> ‘:’ is optional.</td> |
| </tr> |
| <tr> |
| <td><strong>MapType</strong></td> |
| <td>MAP<key_type, value_type></td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| </div> |
| |
| <h3 id="floating-point-special-values">Floating Point Special Values</h3> |
| |
| <p>Spark SQL supports several special floating point values in a case-insensitive manner:</p> |
| |
| <ul> |
| <li>Inf/+Inf/Infinity/+Infinity: positive infinity |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">FloatType</code>: equivalent to Scala <code>Float.PositiveInfinity</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">DoubleType</code>: equivalent to Scala <code>Double.PositiveInfinity</code>.</li> |
| </ul> |
| </li> |
| <li>-Inf/-Infinity: negative infinity |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">FloatType</code>: equivalent to Scala <code>Float.NegativeInfinity</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">DoubleType</code>: equivalent to Scala <code>Double.NegativeInfinity</code>.</li> |
| </ul> |
| </li> |
| <li>NaN: not a number |
| <ul> |
| <li><code class="language-plaintext highlighter-rouge">FloatType</code>: equivalent to Scala <code>Float.NaN</code>.</li> |
| <li><code class="language-plaintext highlighter-rouge">DoubleType</code>: equivalent to Scala <code>Double.NaN</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| |
| <h4 id="positivenegative-infinity-semantics">Positive/Negative Infinity Semantics</h4> |
| |
| <p>There is special handling for positive and negative infinity. They have the following semantics:</p> |
| |
| <ul> |
| <li>Positive infinity multiplied by any positive value returns positive infinity.</li> |
| <li>Negative infinity multiplied by any positive value returns negative infinity.</li> |
| <li>Positive infinity multiplied by any negative value returns negative infinity.</li> |
| <li>Negative infinity multiplied by any negative value returns positive infinity.</li> |
| <li>Positive/negative infinity multiplied by 0 returns NaN.</li> |
| <li>Positive/negative infinity is equal to itself.</li> |
| <li>In aggregations, all positive infinity values are grouped together. Similarly, all negative infinity values are grouped together.</li> |
| <li>Positive infinity and negative infinity are treated as normal values in join keys.</li> |
| <li>Positive infinity sorts lower than NaN and higher than any other values.</li> |
| <li>Negative infinity sorts lower than any other values.</li> |
| </ul> |
| |
| <h4 id="nan-semantics">NaN Semantics</h4> |
| |
| <p>There is special handling for not-a-number (NaN) when dealing with <code class="language-plaintext highlighter-rouge">float</code> or <code class="language-plaintext highlighter-rouge">double</code> types that |
| do not exactly match standard floating point semantics. |
| Specifically:</p> |
| |
| <ul> |
| <li>NaN = NaN returns true.</li> |
| <li>In aggregations, all NaN values are grouped together.</li> |
| <li>NaN is treated as a normal value in join keys.</li> |
| <li>NaN values go last when in ascending order, larger than any other numeric value.</li> |
| </ul> |
| |
| <h4 id="examples">Examples</h4> |
| |
| <div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| <span class="o">|</span><span class="n">Infinity</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">float</span><span class="p">(</span><span class="s1">'-inf'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">---------+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---------+</span> |
| <span class="o">|-</span><span class="n">Infinity</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---------+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">float</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| <span class="o">|</span><span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| <span class="o">|</span><span class="n">NaN</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">)</span> <span class="o">*</span> <span class="mi">0</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| <span class="o">|</span><span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| <span class="o">|</span><span class="n">NaN</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'-infinity'</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="o">-</span><span class="mi">1234567</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| <span class="o">|</span><span class="n">Infinity</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">--------+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">)</span> <span class="o"><</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span><span class="k">true</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">)</span> <span class="o">=</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span><span class="k">true</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| |
| <span class="k">SELECT</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'inf'</span><span class="p">)</span> <span class="o">=</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">)</span> <span class="k">AS</span> <span class="n">col</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span> <span class="n">col</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| <span class="o">|</span><span class="k">true</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">----+</span> |
| |
| <span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">test</span> <span class="p">(</span><span class="n">c1</span> <span class="nb">int</span><span class="p">,</span> <span class="n">c2</span> <span class="nb">double</span><span class="p">);</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'infinity'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'inf'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'-inf'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'NaN'</span><span class="p">));</span> |
| <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">test</span> <span class="k">VALUES</span> <span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="nb">double</span><span class="p">(</span><span class="s1">'-infinity'</span><span class="p">));</span> |
| <span class="k">SELECT</span> <span class="k">COUNT</span><span class="p">(</span><span class="o">*</span><span class="p">),</span> <span class="n">c2</span> <span class="k">FROM</span> <span class="n">test</span> <span class="k">GROUP</span> <span class="k">BY</span> <span class="n">c2</span><span class="p">;</span> |
| <span class="o">+</span><span class="c1">---------+---------+</span> |
| <span class="o">|</span> <span class="k">count</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">|</span> <span class="n">c2</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---------+---------+</span> |
| <span class="o">|</span> <span class="mi">2</span><span class="o">|</span> <span class="n">NaN</span><span class="o">|</span> |
| <span class="o">|</span> <span class="mi">2</span><span class="o">|-</span><span class="n">Infinity</span><span class="o">|</span> |
| <span class="o">|</span> <span class="mi">3</span><span class="o">|</span> <span class="n">Infinity</span><span class="o">|</span> |
| <span class="o">+</span><span class="c1">---------+---------+</span> |
| </code></pre></div></div> |
| |
| |
| </div> |
| |
| <!-- /container --> |
| </div> |
| |
| <script src="js/vendor/jquery-3.5.1.min.js"></script> |
| <script src="js/vendor/bootstrap.bundle.min.js"></script> |
| |
| <script src="js/vendor/anchor.min.js"></script> |
| <script src="js/main.js"></script> |
| |
| <script type="text/javascript" src="js/vendor/docsearch.min.js"></script> |
| <script type="text/javascript"> |
| // DocSearch is entirely free and automated. DocSearch is built in two parts: |
| // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link |
| // in your website and extract content from every page it traverses. It then pushes this |
| // content to an Algolia index. |
| // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index |
| // to your search input and display its results in a dropdown UI. If you want to find more |
| // details on how works DocSearch, check the docs of DocSearch. |
| docsearch({ |
| apiKey: 'd62f962a82bc9abb53471cb7b89da35e', |
| appId: 'RAI69RXRSK', |
| indexName: 'apache_spark', |
| inputSelector: '#docsearch-input', |
| enhancedSearchInput: true, |
| algoliaOptions: { |
| 'facetFilters': ["version:3.5.0"] |
| }, |
| debug: false // Set debug to true if you want to inspect the dropdown |
| }); |
| |
| </script> |
| |
| <!-- MathJax Section --> |
| <script type="text/x-mathjax-config"> |
| MathJax.Hub.Config({ |
| TeX: { equationNumbers: { autoNumber: "AMS" } } |
| }); |
| </script> |
| <script> |
| // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS. |
| // We could use "//cdn.mathjax...", but that won't support "file://". |
| (function(d, script) { |
| script = d.createElement('script'); |
| script.type = 'text/javascript'; |
| script.async = true; |
| script.onload = function(){ |
| MathJax.Hub.Config({ |
| tex2jax: { |
| inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ], |
| displayMath: [ ["$$","$$"], ["\\[", "\\]"] ], |
| processEscapes: true, |
| skipTags: ['script', 'noscript', 'style', 'textarea', 'pre'] |
| } |
| }); |
| }; |
| script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + |
| 'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' + |
| '?config=TeX-AMS-MML_HTMLorMML'; |
| d.getElementsByTagName('head')[0].appendChild(script); |
| }(document)); |
| </script> |
| </body> |
| </html> |
