Publish built docs triggered by 1aa9cb98bc2cc070048027ac96658c69958db516
diff --git a/_sources/autoapi/datafusion/catalog/index.rst.txt b/_sources/autoapi/datafusion/catalog/index.rst.txt index ae8430c..fbf71c6 100644 --- a/_sources/autoapi/datafusion/catalog/index.rst.txt +++ b/_sources/autoapi/datafusion/catalog/index.rst.txt
@@ -73,12 +73,6 @@ This constructor is not typically called by the end user. - .. py:method:: schema() -> pyarrow.Schema - - Returns the schema associated with this table. - - - .. py:property:: kind :type: str @@ -86,6 +80,13 @@ Returns the kind of table. + .. py:property:: schema + :type: pyarrow.Schema + + + Returns the schema associated with this table. + + .. py:attribute:: table
diff --git a/_sources/autoapi/datafusion/context/index.rst.txt b/_sources/autoapi/datafusion/context/index.rst.txt index 2096b7a..97b5848 100644 --- a/_sources/autoapi/datafusion/context/index.rst.txt +++ b/_sources/autoapi/datafusion/context/index.rst.txt
@@ -421,7 +421,7 @@ Create and return a dataframe using the provided partitions. - :param partitions: :py:class:`pyarrow.RecordBatch` partitions to register. + :param partitions: :py:class:`pa.RecordBatch` partitions to register. :param name: Resultant dataframe name. :param schema: Schema for the partitions. @@ -534,6 +534,16 @@ + .. py:method:: global_ctx() -> SessionContext + :classmethod: + + + Retrieve the global context as a `SessionContext` wrapper. + + :returns: A `SessionContext` object that wraps the global `SessionContextInternal`. + + + .. py:method:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = '.avro') -> datafusion.dataframe.DataFrame Create a :py:class:`DataFrame` for reading Avro data source. @@ -586,7 +596,7 @@ - .. py:method:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.Expr]] | None = None) -> datafusion.dataframe.DataFrame + .. py:method:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.Expr | datafusion.expr.SortExpr]] | None = None) -> datafusion.dataframe.DataFrame Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. @@ -657,7 +667,7 @@ .. py:method:: register_dataset(name: str, dataset: pyarrow.dataset.Dataset) -> None - Register a :py:class:`pyarrow.dataset.Dataset` as a table. + Register a :py:class:`pa.dataset.Dataset` as a table. :param name: Name of the table to register. :param dataset: PyArrow dataset. @@ -710,7 +720,7 @@ - .. py:method:: register_parquet(name: str, path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.Expr]] | None = None) -> None + .. py:method:: register_parquet(name: str, path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.SortExpr]] | None = None) -> None Register a Parquet file as a table. @@ -782,6 +792,17 @@ + .. py:method:: register_view(name: str, df: datafusion.dataframe.DataFrame) -> None + + Register a :py:class: `~datafusion.detaframe.DataFrame` as a view. + + :param name: The name to register the view under. + :type name: str + :param df: The DataFrame to be converted into a view and registered. + :type df: DataFrame + + + .. py:method:: session_id() -> str Return an id that uniquely identifies this :py:class:`SessionContext`.
diff --git a/_sources/autoapi/datafusion/dataframe/index.rst.txt b/_sources/autoapi/datafusion/dataframe/index.rst.txt index 9c1dd56..200e796 100644 --- a/_sources/autoapi/datafusion/dataframe/index.rst.txt +++ b/_sources/autoapi/datafusion/dataframe/index.rst.txt
@@ -115,7 +115,7 @@ - .. py:method:: __getitem__(key: str | List[str]) -> DataFrame + .. py:method:: __getitem__(key: str | list[str]) -> DataFrame Return a new :py:class`DataFrame` with the specified column or columns. @@ -319,6 +319,12 @@ + .. py:method:: into_view() -> pyarrow.Table + + Convert DataFrame as a ViewTable which can be used in register_table. + + + .. py:method:: join(right: DataFrame, on: str | Sequence[str], how: Literal['inner', 'left', 'right', 'full', 'semi', 'anti'] = 'inner', *, left_on: None = None, right_on: None = None, join_keys: None = None) -> DataFrame join(right: DataFrame, on: None = None, how: Literal['inner', 'left', 'right', 'full', 'semi', 'anti'] = 'inner', *, left_on: str | Sequence[str], right_on: str | Sequence[str], join_keys: tuple[list[str], list[str]] | None = None) -> DataFrame join(right: DataFrame, on: None = None, how: Literal['inner', 'left', 'right', 'full', 'semi', 'anti'] = 'inner', *, join_keys: tuple[list[str], list[str]], left_on: None = None, right_on: None = None) -> DataFrame
diff --git a/_sources/autoapi/datafusion/expr/index.rst.txt b/_sources/autoapi/datafusion/expr/index.rst.txt index a6e6abb..72d4cdc 100644 --- a/_sources/autoapi/datafusion/expr/index.rst.txt +++ b/_sources/autoapi/datafusion/expr/index.rst.txt
@@ -132,7 +132,7 @@ .. py:attribute:: case_builder -.. py:class:: Expr(expr: datafusion._internal.expr.Expr) +.. py:class:: Expr(expr: datafusion._internal.expr.RawExpr) Expression object. @@ -156,7 +156,7 @@ - .. py:method:: __eq__(rhs: Any) -> Expr + .. py:method:: __eq__(rhs: object) -> Expr Equal to. @@ -228,7 +228,7 @@ - .. py:method:: __ne__(rhs: Any) -> Expr + .. py:method:: __ne__(rhs: object) -> Expr Not equal to. @@ -292,7 +292,7 @@ - .. py:method:: cast(to: pyarrow.DataType[Any] | Type[float] | Type[int] | Type[str] | Type[bool]) -> Expr + .. py:method:: cast(to: pyarrow.DataType[Any] | type[float | int | str | bool]) -> Expr Cast to a new data type. @@ -540,6 +540,7 @@ .. py:attribute:: _to_pyarrow_types + :type: ClassVar[dict[type, pyarrow.DataType]] .. py:attribute:: expr @@ -638,7 +639,7 @@ - .. py:method:: get_upper_bound() + .. py:method:: get_upper_bound() -> WindowFrameBound Returns end bound.
diff --git a/_sources/autoapi/datafusion/functions/index.rst.txt b/_sources/autoapi/datafusion/functions/index.rst.txt index 2ae0836..21fa1d2 100644 --- a/_sources/autoapi/datafusion/functions/index.rst.txt +++ b/_sources/autoapi/datafusion/functions/index.rst.txt
@@ -194,6 +194,7 @@ datafusion.functions.random datafusion.functions.range datafusion.functions.rank + datafusion.functions.regexp_count datafusion.functions.regexp_like datafusion.functions.regexp_match datafusion.functions.regexp_replace @@ -240,6 +241,7 @@ datafusion.functions.to_timestamp datafusion.functions.to_timestamp_micros datafusion.functions.to_timestamp_millis + datafusion.functions.to_timestamp_nanos datafusion.functions.to_timestamp_seconds datafusion.functions.to_unixtime datafusion.functions.translate @@ -935,7 +937,7 @@ This is an alias for :py:func:`date_trunc`. -.. py:function:: decode(input: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr +.. py:function:: decode(expr: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr Decode the ``input``, using the ``encoding``. encoding can be base64 or hex. @@ -979,7 +981,7 @@ This is an alias for :py:func:`array_empty`. -.. py:function:: encode(input: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr +.. py:function:: encode(expr: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr Encode the ``input``, using the ``encoding``. encoding can be base64 or hex. @@ -1655,6 +1657,14 @@ :param order_by: Set ordering within the window frame. +.. py:function:: regexp_count(string: datafusion.expr.Expr, pattern: datafusion.expr.Expr, start: datafusion.expr.Expr, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr + + Returns the number of matches in a string. + + Optional start position (the first position is 1) to search for the regular + expression. + + .. py:function:: regexp_like(string: datafusion.expr.Expr, regex: datafusion.expr.Expr, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr Find if any regular expression (regex) matches exist. @@ -1837,7 +1847,7 @@ Returns the last ``n`` characters in the ``string``. -.. py:function:: round(value: datafusion.expr.Expr, decimal_places: datafusion.expr.Expr = Expr.literal(0)) -> datafusion.expr.Expr +.. py:function:: round(value: datafusion.expr.Expr, decimal_places: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr Round the argument to the nearest integer. @@ -2058,6 +2068,13 @@ See :py:func:`to_timestamp` for a description on how to use formatters. +.. py:function:: to_timestamp_nanos(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr + + Converts a string and optional formats to a ``Timestamp`` in nanoseconds. + + See :py:func:`to_timestamp` for a description on how to use formatters. + + .. py:function:: to_timestamp_seconds(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr Converts a string and optional formats to a ``Timestamp`` in seconds. @@ -2090,7 +2107,7 @@ Converts a string to uppercase. -.. py:function:: uuid(arg: datafusion.expr.Expr) -> datafusion.expr.Expr +.. py:function:: uuid() -> datafusion.expr.Expr Returns uuid v4 as a string value.
diff --git a/_sources/autoapi/datafusion/index.rst.txt b/_sources/autoapi/datafusion/index.rst.txt index 35db823..84313ec 100644 --- a/_sources/autoapi/datafusion/index.rst.txt +++ b/_sources/autoapi/datafusion/index.rst.txt
@@ -24,6 +24,7 @@ /autoapi/datafusion/expr/index /autoapi/datafusion/functions/index /autoapi/datafusion/input/index + /autoapi/datafusion/io/index /autoapi/datafusion/object_store/index /autoapi/datafusion/plan/index /autoapi/datafusion/record_batch/index @@ -37,6 +38,9 @@ .. autoapisummary:: datafusion.DFSchema + datafusion.udaf + datafusion.udf + datafusion.udwf Classes @@ -71,6 +75,10 @@ datafusion.column datafusion.lit datafusion.literal + datafusion.read_avro + datafusion.read_csv + datafusion.read_json + datafusion.read_parquet Package Contents @@ -89,7 +97,7 @@ - .. py:method:: merge(states: List[pyarrow.Array]) -> None + .. py:method:: merge(states: list[pyarrow.Array]) -> None :abstractmethod: @@ -97,7 +105,7 @@ - .. py:method:: state() -> List[pyarrow.Scalar] + .. py:method:: state() -> list[pyarrow.Scalar] :abstractmethod: @@ -113,7 +121,7 @@ -.. py:class:: AggregateUDF(name: Optional[str], accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str) +.. py:class:: AggregateUDF(name: str, accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str) Class for performing scalar user-defined functions (UDF). @@ -135,20 +143,31 @@ - .. py:method:: udaf(accum: Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> AggregateUDF + .. py:method:: udaf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, AggregateUDF] + udaf(accum: Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> AggregateUDF :staticmethod: - Create a new User-Defined Aggregate Function. + Create a new User-Defined Aggregate Function (UDAF). - If your :py:class:`Accumulator` can be instantiated with no arguments, you - can simply pass it's type as ``accum``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`Accumulator` will be constructed for every instance in - which this UDAF is used. The following examples are all valid. + This class allows you to define an **aggregate function** that can be used in + data aggregation or window function calls. - .. code-block:: python + Usage: + - **As a function**: Call `udaf(accum, input_types, return_type, state_type, + volatility, name)`. + - **As a decorator**: Use `@udaf(input_types, return_type, state_type, + volatility, name)`. + When using `udaf` as a decorator, **do not pass `accum` explicitly**. + **Function example:** + + If your `:py:class:Accumulator` can be instantiated with no arguments, you + can simply pass it's type as `accum`. If you need to pass additional + arguments to it's constructor, you can define a lambda or a factory method. + During runtime the `:py:class:Accumulator` will be constructed for every + instance in which this UDAF is used. The following examples are all valid. + ``` import pyarrow as pa import pyarrow.compute as pc @@ -156,13 +175,13 @@ def __init__(self, bias: float = 0.0): self._sum = pa.scalar(bias) - def state(self) -> List[pa.Scalar]: + def state(self) -> list[pa.Scalar]: return [self._sum] def update(self, values: pa.Array) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) - def merge(self, states: List[pa.Array]) -> None: + def merge(self, states: list[pa.Array]) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) def evaluate(self) -> pa.Scalar: @@ -171,11 +190,23 @@ def sum_bias_10() -> Summarize: return Summarize(10.0) - udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), [pa.float64()], "immutable") + udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), + [pa.float64()], "immutable") + ``` - :param accum: The accumulator python function. + **Decorator example:** + ``` + @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable") + def udf4() -> Summarize: + return Summarize(10.0) + ``` + + :param accum: The accumulator python function. **Only needed when calling as a + function. Skip this argument when using `udaf` as a decorator.** :param input_types: The data types of the arguments to ``accum``. :param return_type: The data type of the return value. :param state_type: The data types of the intermediate accumulation. @@ -247,7 +278,7 @@ - .. py:method:: children() -> List[ExecutionPlan] + .. py:method:: children() -> list[ExecutionPlan] Get a list of children `ExecutionPlan` that act as inputs to this plan. @@ -296,7 +327,7 @@ Returns the number of partitions in the physical plan. -.. py:class:: Expr(expr: datafusion._internal.expr.Expr) +.. py:class:: Expr(expr: datafusion._internal.expr.RawExpr) Expression object. @@ -320,7 +351,7 @@ - .. py:method:: __eq__(rhs: Any) -> Expr + .. py:method:: __eq__(rhs: object) -> Expr Equal to. @@ -392,7 +423,7 @@ - .. py:method:: __ne__(rhs: Any) -> Expr + .. py:method:: __ne__(rhs: object) -> Expr Not equal to. @@ -456,7 +487,7 @@ - .. py:method:: cast(to: pyarrow.DataType[Any] | Type[float] | Type[int] | Type[str] | Type[bool]) -> Expr + .. py:method:: cast(to: pyarrow.DataType[Any] | type[float | int | str | bool]) -> Expr Cast to a new data type. @@ -704,6 +735,7 @@ .. py:attribute:: _to_pyarrow_types + :type: ClassVar[dict[type, pyarrow.DataType]] .. py:attribute:: expr @@ -770,7 +802,7 @@ - .. py:method:: inputs() -> List[LogicalPlan] + .. py:method:: inputs() -> list[LogicalPlan] Returns the list of inputs to the logical plan. @@ -795,7 +827,7 @@ .. py:class:: RecordBatch(record_batch: datafusion._internal.RecordBatch) - This class is essentially a wrapper for :py:class:`pyarrow.RecordBatch`. + This class is essentially a wrapper for :py:class:`pa.RecordBatch`. This constructor is generally not called by the end user. @@ -804,7 +836,7 @@ .. py:method:: to_pyarrow() -> pyarrow.RecordBatch - Convert to :py:class:`pyarrow.RecordBatch`. + Convert to :py:class:`pa.RecordBatch`. @@ -1022,7 +1054,7 @@ .. py:attribute:: options_internal -.. py:class:: ScalarUDF(name: Optional[str], func: Callable[Ellipsis, _R], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, volatility: Volatility | str) +.. py:class:: ScalarUDF(name: str, func: Callable[Ellipsis, _R], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, volatility: Volatility | str) Class for performing scalar user-defined functions (UDF). @@ -1043,24 +1075,54 @@ - .. py:method:: udf(func: Callable[Ellipsis, _R], input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> ScalarUDF + .. py:method:: udf(input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, ScalarUDF] + udf(func: Callable[Ellipsis, _R], input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> ScalarUDF :staticmethod: - Create a new User-Defined Function. + Create a new User-Defined Function (UDF). - :param func: A callable python function. - :param input_types: The data types of the arguments to ``func``. This list - must be of the same length as the number of arguments. - :param return_type: The data type of the return value from the python - function. - :param volatility: See ``Volatility`` for allowed values. + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udf(input_types, return_type, volatility, + name)`. In this case, do **not** pass `func` explicitly. + + :param func: **Only needed when calling as a function.** + Skip this argument when using `udf` as a decorator. + :type func: Callable, optional + :param input_types: The data types of the arguments + to `func`. This list must be of the same length as the number of + arguments. + :type input_types: list[pa.DataType] + :param return_type: The data type of the return value from the function. + :type return_type: _R + :param volatility: See `Volatility` for allowed values. + :type volatility: Volatility | str :param name: A descriptive name for the function. + :type name: Optional[str] - :returns: + :returns: A user-defined function that can be used in SQL expressions, + data aggregation, or window function calls. - A user-defined aggregate function, which can be used in either data - aggregation or window function calls. + .. rubric:: Example + + **Using `udf` as a function:** + ``` + def double_func(x): + return x * 2 + double_udf = udf(double_func, [pa.int32()], pa.int32(), + "volatile", "double_it") + ``` + + **Using `udf` as a decorator:** + ``` + @udf([pa.int32()], pa.int32(), "volatile", "double_it") + def double_udf(x): + return x * 2 + ``` @@ -1230,12 +1292,6 @@ This constructor is not typically called by the end user. - .. py:method:: schema() -> pyarrow.Schema - - Returns the schema associated with this table. - - - .. py:property:: kind :type: str @@ -1243,6 +1299,13 @@ Returns the kind of table. + .. py:property:: schema + :type: pyarrow.Schema + + + Returns the schema associated with this table. + + .. py:attribute:: table @@ -1273,7 +1336,7 @@ - .. py:method:: get_upper_bound() + .. py:method:: get_upper_bound() -> WindowFrameBound Returns end bound. @@ -1282,7 +1345,7 @@ .. py:attribute:: window_frame -.. py:class:: WindowUDF(name: Optional[str], func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str) +.. py:class:: WindowUDF(name: str, func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str) Class for performing window user-defined functions (UDF). @@ -1304,27 +1367,64 @@ - .. py:method:: udwf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF + .. py:method:: _create_window_udf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF :staticmethod: - Create a new User-Defined Window Function. + Create a WindowUDF instance from function arguments. - If your :py:class:`WindowEvaluator` can be instantiated with no arguments, you - can simply pass it's type as ``func``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`WindowEvaluator` will be constructed for every instance in - which this UDWF is used. The following examples are all valid. - .. code-block:: python + .. py:method:: _create_window_udf_decorator(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> Callable[[Callable[[], WindowEvaluator]], Callable[Ellipsis, datafusion.expr.Expr]] + :staticmethod: + + + Create a decorator for a WindowUDF. + + + + .. py:method:: _get_default_name(func: Callable) -> str + :staticmethod: + + + Get the default name for a function based on its attributes. + + + + .. py:method:: _normalize_input_types(input_types: pyarrow.DataType | list[pyarrow.DataType]) -> list[pyarrow.DataType] + :staticmethod: + + + Convert a single DataType to a list if needed. + + + + .. py:method:: udwf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, WindowUDF] + udwf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF + :staticmethod: + + + Create a new User-Defined Window Function (UDWF). + + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udwf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udwf(input_types, return_type, volatility, + name)`. When using `udwf` as a decorator, **do not pass `func` + explicitly**. + + **Function example:** + ``` import pyarrow as pa class BiasedNumbers(WindowEvaluator): def __init__(self, start: int = 0) -> None: self.start = start - def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + def evaluate_all(self, values: list[pa.Array], + num_rows: int) -> pa.Array: return pa.array([self.start + i for i in range(num_rows)]) def bias_10() -> BiasedNumbers: @@ -1334,39 +1434,140 @@ udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable") udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable") - :param func: A callable to create the window function. - :param input_types: The data types of the arguments to ``func``. + ``` + + **Decorator example:** + ``` + @udwf(pa.int64(), pa.int64(), "immutable") + def biased_numbers() -> BiasedNumbers: + return BiasedNumbers(10) + ``` + + :param func: **Only needed when calling as a function. Skip this argument when + using `udwf` as a decorator.** + :param input_types: The data types of the arguments. :param return_type: The data type of the return value. :param volatility: See :py:class:`Volatility` for allowed values. - :param arguments: A list of arguments to pass in to the __init__ method for accum. :param name: A descriptive name for the function. - :returns: A user-defined window function. + :returns: A user-defined window function that can be used in window function calls. .. py:attribute:: _udwf -.. py:function:: col(value: str) +.. py:function:: col(value: str) -> expr.Expr Create a column expression. -.. py:function:: column(value: str) +.. py:function:: column(value: str) -> expr.Expr Create a column expression. -.. py:function:: lit(value) +.. py:function:: lit(value) -> expr.Expr Create a literal expression. -.. py:function:: literal(value) +.. py:function:: literal(value) -> expr.Expr Create a literal expression. +.. py:function:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = '.avro') -> datafusion.dataframe.DataFrame + + Create a :py:class:`DataFrame` for reading Avro data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the Avro file. + :param schema: The data source schema. + :param file_partition_cols: Partition columns. + :param file_extension: File extension to select. + + :returns: DataFrame representation of the read Avro file + + +.. py:function:: read_csv(path: str | pathlib.Path | list[str] | list[pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = 1000, file_extension: str = '.csv', table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame + + Read a CSV data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the CSV file + :param schema: An optional schema representing the CSV files. If None, the + CSV reader will try to infer it based on data in file. + :param has_header: Whether the CSV file have a header. If schema inference + is run on a file with no headers, default column names are + created. + :param delimiter: An optional column delimiter. + :param schema_infer_max_records: Maximum number of rows to read from CSV + files for schema inference if needed. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param table_partition_cols: Partition columns. + :param file_compression_type: File compression type. + + :returns: DataFrame representation of the read CSV files + + +.. py:function:: read_json(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame + + Read a line-delimited JSON data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the JSON file. + :param schema: The data source schema. + :param schema_infer_max_records: Maximum number of rows to read from JSON + files for schema inference if needed. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param table_partition_cols: Partition columns. + :param file_compression_type: File compression type. + + :returns: DataFrame representation of the read JSON files. + + +.. py:function:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.Expr]] | None = None) -> datafusion.dataframe.DataFrame + + Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the Parquet file. + :param table_partition_cols: Partition columns. + :param parquet_pruning: Whether the parquet reader should use the predicate + to prune row groups. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param skip_metadata: Whether the parquet reader should skip any metadata + that may be in the file schema. This can help avoid schema + conflicts due to metadata. + :param schema: An optional schema representing the parquet files. If None, + the parquet reader will try to infer it based on data in the + file. + :param file_sort_order: Sort order for the file. + + :returns: DataFrame representation of the read Parquet files + + .. py:data:: DFSchema +.. py:data:: udaf + +.. py:data:: udf + +.. py:data:: udwf +
diff --git a/_sources/autoapi/datafusion/input/base/index.rst.txt b/_sources/autoapi/datafusion/input/base/index.rst.txt index 7deb151..9b962d0 100644 --- a/_sources/autoapi/datafusion/input/base/index.rst.txt +++ b/_sources/autoapi/datafusion/input/base/index.rst.txt
@@ -37,7 +37,7 @@ to obtain the SqlTable information from the custom datasource. - .. py:method:: build_table(input_item: Any, table_name: str, **kwarg) -> datafusion.common.SqlTable + .. py:method:: build_table(input_item: Any, table_name: str, **kwarg: Any) -> datafusion.common.SqlTable :abstractmethod: @@ -45,7 +45,7 @@ - .. py:method:: is_correct_input(input_item: Any, table_name: str, **kwargs) -> bool + .. py:method:: is_correct_input(input_item: Any, table_name: str, **kwargs: Any) -> bool :abstractmethod:
diff --git a/_sources/autoapi/datafusion/input/index.rst.txt b/_sources/autoapi/datafusion/input/index.rst.txt index 27ebad0..9a081ed 100644 --- a/_sources/autoapi/datafusion/input/index.rst.txt +++ b/_sources/autoapi/datafusion/input/index.rst.txt
@@ -21,3 +21,36 @@ /autoapi/datafusion/input/location/index +Classes +------- + +.. autoapisummary:: + + datafusion.input.LocationInputPlugin + + +Package Contents +---------------- + +.. py:class:: LocationInputPlugin + + Bases: :py:obj:`datafusion.input.base.BaseInputSource` + + + Input Plugin for everything. + + This can be read in from a file (on disk, remote etc.). + + + .. py:method:: build_table(input_item: str, table_name: str, **kwargs: Any) -> datafusion.common.SqlTable + + Create a table from the input source. + + + + .. py:method:: is_correct_input(input_item: Any, table_name: str, **kwargs: Any) -> bool + + Returns `True` if the input is valid. + + +
diff --git a/_sources/autoapi/datafusion/input/location/index.rst.txt b/_sources/autoapi/datafusion/input/location/index.rst.txt index 195c81a..609a280 100644 --- a/_sources/autoapi/datafusion/input/location/index.rst.txt +++ b/_sources/autoapi/datafusion/input/location/index.rst.txt
@@ -30,13 +30,13 @@ This can be read in from a file (on disk, remote etc.). - .. py:method:: build_table(input_file: str, table_name: str, **kwargs) -> datafusion.common.SqlTable + .. py:method:: build_table(input_item: str, table_name: str, **kwargs: Any) -> datafusion.common.SqlTable Create a table from the input source. - .. py:method:: is_correct_input(input_item: Any, table_name: str, **kwargs) + .. py:method:: is_correct_input(input_item: Any, table_name: str, **kwargs: Any) -> bool Returns `True` if the input is valid.
diff --git a/_sources/autoapi/datafusion/io/index.rst.txt b/_sources/autoapi/datafusion/io/index.rst.txt new file mode 100644 index 0000000..79ef20e --- /dev/null +++ b/_sources/autoapi/datafusion/io/index.rst.txt
@@ -0,0 +1,111 @@ +datafusion.io +============= + +.. py:module:: datafusion.io + +.. autoapi-nested-parse:: + + IO read functions using global context. + + + +Functions +--------- + +.. autoapisummary:: + + datafusion.io.read_avro + datafusion.io.read_csv + datafusion.io.read_json + datafusion.io.read_parquet + + +Module Contents +--------------- + +.. py:function:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = '.avro') -> datafusion.dataframe.DataFrame + + Create a :py:class:`DataFrame` for reading Avro data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the Avro file. + :param schema: The data source schema. + :param file_partition_cols: Partition columns. + :param file_extension: File extension to select. + + :returns: DataFrame representation of the read Avro file + + +.. py:function:: read_csv(path: str | pathlib.Path | list[str] | list[pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = 1000, file_extension: str = '.csv', table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame + + Read a CSV data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the CSV file + :param schema: An optional schema representing the CSV files. If None, the + CSV reader will try to infer it based on data in file. + :param has_header: Whether the CSV file have a header. If schema inference + is run on a file with no headers, default column names are + created. + :param delimiter: An optional column delimiter. + :param schema_infer_max_records: Maximum number of rows to read from CSV + files for schema inference if needed. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param table_partition_cols: Partition columns. + :param file_compression_type: File compression type. + + :returns: DataFrame representation of the read CSV files + + +.. py:function:: read_json(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame + + Read a line-delimited JSON data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the JSON file. + :param schema: The data source schema. + :param schema_infer_max_records: Maximum number of rows to read from JSON + files for schema inference if needed. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param table_partition_cols: Partition columns. + :param file_compression_type: File compression type. + + :returns: DataFrame representation of the read JSON files. + + +.. py:function:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: list[list[datafusion.expr.Expr]] | None = None) -> datafusion.dataframe.DataFrame + + Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + :param path: Path to the Parquet file. + :param table_partition_cols: Partition columns. + :param parquet_pruning: Whether the parquet reader should use the predicate + to prune row groups. + :param file_extension: File extension; only files with this extension are + selected for data input. + :param skip_metadata: Whether the parquet reader should skip any metadata + that may be in the file schema. This can help avoid schema + conflicts due to metadata. + :param schema: An optional schema representing the parquet files. If None, + the parquet reader will try to infer it based on data in the + file. + :param file_sort_order: Sort order for the file. + + :returns: DataFrame representation of the read Parquet files + +
diff --git a/_sources/autoapi/datafusion/plan/index.rst.txt b/_sources/autoapi/datafusion/plan/index.rst.txt index 5434d35..c29dcda 100644 --- a/_sources/autoapi/datafusion/plan/index.rst.txt +++ b/_sources/autoapi/datafusion/plan/index.rst.txt
@@ -34,7 +34,7 @@ - .. py:method:: children() -> List[ExecutionPlan] + .. py:method:: children() -> list[ExecutionPlan] Get a list of children `ExecutionPlan` that act as inputs to this plan. @@ -144,7 +144,7 @@ - .. py:method:: inputs() -> List[LogicalPlan] + .. py:method:: inputs() -> list[LogicalPlan] Returns the list of inputs to the logical plan.
diff --git a/_sources/autoapi/datafusion/record_batch/index.rst.txt b/_sources/autoapi/datafusion/record_batch/index.rst.txt index 529dafd..ae91b4c 100644 --- a/_sources/autoapi/datafusion/record_batch/index.rst.txt +++ b/_sources/autoapi/datafusion/record_batch/index.rst.txt
@@ -26,7 +26,7 @@ .. py:class:: RecordBatch(record_batch: datafusion._internal.RecordBatch) - This class is essentially a wrapper for :py:class:`pyarrow.RecordBatch`. + This class is essentially a wrapper for :py:class:`pa.RecordBatch`. This constructor is generally not called by the end user. @@ -35,7 +35,7 @@ .. py:method:: to_pyarrow() -> pyarrow.RecordBatch - Convert to :py:class:`pyarrow.RecordBatch`. + Convert to :py:class:`pa.RecordBatch`.
diff --git a/_sources/autoapi/datafusion/udf/index.rst.txt b/_sources/autoapi/datafusion/udf/index.rst.txt index 443a844..882bbcd 100644 --- a/_sources/autoapi/datafusion/udf/index.rst.txt +++ b/_sources/autoapi/datafusion/udf/index.rst.txt
@@ -15,7 +15,9 @@ .. autoapisummary:: datafusion.udf._R - datafusion.udf._W + datafusion.udf.udaf + datafusion.udf.udf + datafusion.udf.udwf Classes @@ -47,7 +49,7 @@ - .. py:method:: merge(states: List[pyarrow.Array]) -> None + .. py:method:: merge(states: list[pyarrow.Array]) -> None :abstractmethod: @@ -55,7 +57,7 @@ - .. py:method:: state() -> List[pyarrow.Scalar] + .. py:method:: state() -> list[pyarrow.Scalar] :abstractmethod: @@ -71,7 +73,7 @@ -.. py:class:: AggregateUDF(name: Optional[str], accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str) +.. py:class:: AggregateUDF(name: str, accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str) Class for performing scalar user-defined functions (UDF). @@ -93,20 +95,31 @@ - .. py:method:: udaf(accum: Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> AggregateUDF + .. py:method:: udaf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, AggregateUDF] + udaf(accum: Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: Optional[str] = None) -> AggregateUDF :staticmethod: - Create a new User-Defined Aggregate Function. + Create a new User-Defined Aggregate Function (UDAF). - If your :py:class:`Accumulator` can be instantiated with no arguments, you - can simply pass it's type as ``accum``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`Accumulator` will be constructed for every instance in - which this UDAF is used. The following examples are all valid. + This class allows you to define an **aggregate function** that can be used in + data aggregation or window function calls. - .. code-block:: python + Usage: + - **As a function**: Call `udaf(accum, input_types, return_type, state_type, + volatility, name)`. + - **As a decorator**: Use `@udaf(input_types, return_type, state_type, + volatility, name)`. + When using `udaf` as a decorator, **do not pass `accum` explicitly**. + **Function example:** + + If your `:py:class:Accumulator` can be instantiated with no arguments, you + can simply pass it's type as `accum`. If you need to pass additional + arguments to it's constructor, you can define a lambda or a factory method. + During runtime the `:py:class:Accumulator` will be constructed for every + instance in which this UDAF is used. The following examples are all valid. + ``` import pyarrow as pa import pyarrow.compute as pc @@ -114,13 +127,13 @@ def __init__(self, bias: float = 0.0): self._sum = pa.scalar(bias) - def state(self) -> List[pa.Scalar]: + def state(self) -> list[pa.Scalar]: return [self._sum] def update(self, values: pa.Array) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) - def merge(self, states: List[pa.Array]) -> None: + def merge(self, states: list[pa.Array]) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) def evaluate(self) -> pa.Scalar: @@ -129,11 +142,23 @@ def sum_bias_10() -> Summarize: return Summarize(10.0) - udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), [pa.float64()], "immutable") + udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), + [pa.float64()], "immutable") + ``` - :param accum: The accumulator python function. + **Decorator example:** + ``` + @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable") + def udf4() -> Summarize: + return Summarize(10.0) + ``` + + :param accum: The accumulator python function. **Only needed when calling as a + function. Skip this argument when using `udaf` as a decorator.** :param input_types: The data types of the arguments to ``accum``. :param return_type: The data type of the return value. :param state_type: The data types of the intermediate accumulation. @@ -148,7 +173,7 @@ .. py:attribute:: _udaf -.. py:class:: ScalarUDF(name: Optional[str], func: Callable[Ellipsis, _R], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, volatility: Volatility | str) +.. py:class:: ScalarUDF(name: str, func: Callable[Ellipsis, _R], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, volatility: Volatility | str) Class for performing scalar user-defined functions (UDF). @@ -169,24 +194,54 @@ - .. py:method:: udf(func: Callable[Ellipsis, _R], input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> ScalarUDF + .. py:method:: udf(input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, ScalarUDF] + udf(func: Callable[Ellipsis, _R], input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None) -> ScalarUDF :staticmethod: - Create a new User-Defined Function. + Create a new User-Defined Function (UDF). - :param func: A callable python function. - :param input_types: The data types of the arguments to ``func``. This list - must be of the same length as the number of arguments. - :param return_type: The data type of the return value from the python - function. - :param volatility: See ``Volatility`` for allowed values. + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udf(input_types, return_type, volatility, + name)`. In this case, do **not** pass `func` explicitly. + + :param func: **Only needed when calling as a function.** + Skip this argument when using `udf` as a decorator. + :type func: Callable, optional + :param input_types: The data types of the arguments + to `func`. This list must be of the same length as the number of + arguments. + :type input_types: list[pa.DataType] + :param return_type: The data type of the return value from the function. + :type return_type: _R + :param volatility: See `Volatility` for allowed values. + :type volatility: Volatility | str :param name: A descriptive name for the function. + :type name: Optional[str] - :returns: + :returns: A user-defined function that can be used in SQL expressions, + data aggregation, or window function calls. - A user-defined aggregate function, which can be used in either data - aggregation or window function calls. + .. rubric:: Example + + **Using `udf` as a function:** + ``` + def double_func(x): + return x * 2 + double_udf = udf(double_func, [pa.int32()], pa.int32(), + "volatile", "double_it") + ``` + + **Using `udf` as a decorator:** + ``` + @udf([pa.int32()], pa.int32(), "volatile", "double_it") + def double_udf(x): + return x * 2 + ``` @@ -205,7 +260,7 @@ name (`"immutable"`, `"stable"`, or `"volatile"`). - .. py:method:: __str__() + .. py:method:: __str__() -> str Returns the string equivalent. @@ -293,9 +348,12 @@ This function is called once per input *partition* for window functions that *do not use* values from the window frame, such as - :py:func:`~datafusion.functions.row_number`, :py:func:`~datafusion.functions.rank`, - :py:func:`~datafusion.functions.dense_rank`, :py:func:`~datafusion.functions.percent_rank`, - :py:func:`~datafusion.functions.cume_dist`, :py:func:`~datafusion.functions.lead`, + :py:func:`~datafusion.functions.row_number`, + :py:func:`~datafusion.functions.rank`, + :py:func:`~datafusion.functions.dense_rank`, + :py:func:`~datafusion.functions.percent_rank`, + :py:func:`~datafusion.functions.cume_dist`, + :py:func:`~datafusion.functions.lead`, and :py:func:`~datafusion.functions.lag`. It produces the result of all rows in a single pass. It @@ -418,7 +476,7 @@ -.. py:class:: WindowUDF(name: Optional[str], func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str) +.. py:class:: WindowUDF(name: str, func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str) Class for performing window user-defined functions (UDF). @@ -440,27 +498,64 @@ - .. py:method:: udwf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF + .. py:method:: _create_window_udf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF :staticmethod: - Create a new User-Defined Window Function. + Create a WindowUDF instance from function arguments. - If your :py:class:`WindowEvaluator` can be instantiated with no arguments, you - can simply pass it's type as ``func``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`WindowEvaluator` will be constructed for every instance in - which this UDWF is used. The following examples are all valid. - .. code-block:: python + .. py:method:: _create_window_udf_decorator(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> Callable[[Callable[[], WindowEvaluator]], Callable[Ellipsis, datafusion.expr.Expr]] + :staticmethod: + + + Create a decorator for a WindowUDF. + + + + .. py:method:: _get_default_name(func: Callable) -> str + :staticmethod: + + + Get the default name for a function based on its attributes. + + + + .. py:method:: _normalize_input_types(input_types: pyarrow.DataType | list[pyarrow.DataType]) -> list[pyarrow.DataType] + :staticmethod: + + + Convert a single DataType to a list if needed. + + + + .. py:method:: udwf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> Callable[Ellipsis, WindowUDF] + udwf(func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: Optional[str] = None) -> WindowUDF + :staticmethod: + + + Create a new User-Defined Window Function (UDWF). + + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udwf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udwf(input_types, return_type, volatility, + name)`. When using `udwf` as a decorator, **do not pass `func` + explicitly**. + + **Function example:** + ``` import pyarrow as pa class BiasedNumbers(WindowEvaluator): def __init__(self, start: int = 0) -> None: self.start = start - def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + def evaluate_all(self, values: list[pa.Array], + num_rows: int) -> pa.Array: return pa.array([self.start + i for i in range(num_rows)]) def bias_10() -> BiasedNumbers: @@ -470,14 +565,23 @@ udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable") udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable") - :param func: A callable to create the window function. - :param input_types: The data types of the arguments to ``func``. + ``` + + **Decorator example:** + ``` + @udwf(pa.int64(), pa.int64(), "immutable") + def biased_numbers() -> BiasedNumbers: + return BiasedNumbers(10) + ``` + + :param func: **Only needed when calling as a function. Skip this argument when + using `udwf` as a decorator.** + :param input_types: The data types of the arguments. :param return_type: The data type of the return value. :param volatility: See :py:class:`Volatility` for allowed values. - :param arguments: A list of arguments to pass in to the __init__ method for accum. :param name: A descriptive name for the function. - :returns: A user-defined window function. + :returns: A user-defined window function that can be used in window function calls. @@ -486,5 +590,9 @@ .. py:data:: _R -.. py:data:: _W +.. py:data:: udaf + +.. py:data:: udf + +.. py:data:: udwf
diff --git a/_sources/contributor-guide/ffi.rst.txt b/_sources/contributor-guide/ffi.rst.txt new file mode 100644 index 0000000..c1f9806 --- /dev/null +++ b/_sources/contributor-guide/ffi.rst.txt
@@ -0,0 +1,212 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Python Extensions +================= + +The DataFusion in Python project is designed to allow users to extend its functionality in a few core +areas. Ideally many users would like to package their extensions as a Python package and easily +integrate that package with this project. This page serves to describe some of the challenges we face +when doing these integrations and the approach our project uses. + +The Primary Issue +----------------- + +Suppose you wish to use DataFusion and you have a custom data source that can produce tables that +can then be queried against, similar to how you can register a :ref:`CSV <io_csv>` or +:ref:`Parquet <io_parquet>` file. In DataFusion terminology, you likely want to implement a +:ref:`Custom Table Provider <io_custom_table_provider>`. In an effort to make your data source +as performant as possible and to utilize the features of DataFusion, you may decide to write +your source in Rust and then expose it through `PyO3 <https://pyo3.rs>`_ as a Python library. + +At first glance, it may appear the best way to do this is to add the ``datafusion-python`` +crate as a dependency, provide a ``PyTable``, and then to register it with the +``SessionContext``. Unfortunately, this will not work. + +When you produce your code as a Python library and it needs to interact with the DataFusion +library, at the lowest level they communicate through an Application Binary Interface (ABI). +The acronym sounds similar to API (Application Programming Interface), but it is distinctly +different. + +The ABI sets the standard for how these libraries can share data and functions between each +other. One of the key differences between Rust and other programming languages is that Rust +does not have a stable ABI. What this means in practice is that if you compile a Rust library +with one version of the ``rustc`` compiler and I compile another library to interface with it +but I use a different version of the compiler, there is no guarantee the interface will be +the same. + +In practice, this means that a Python library built with ``datafusion-python`` as a Rust +dependency will generally **not** be compatible with the DataFusion Python package, even +if they reference the same version of ``datafusion-python``. If you attempt to do this, it may +work on your local computer if you have built both packages with the same optimizations. +This can sometimes lead to a false expectation that the code will work, but it frequently +breaks the moment you try to use your package against the released packages. + +You can find more information about the Rust ABI in their +`online documentation <https://doc.rust-lang.org/reference/abi.html>`_. + +The FFI Approach +---------------- + +Rust supports interacting with other programming languages through it's Foreign Function +Interface (FFI). The advantage of using the FFI is that it enables you to write data structures +and functions that have a stable ABI. The allows you to use Rust code with C, Python, and +other languages. In fact, the `PyO3 <https://pyo3.rs>`_ library uses the FFI to share data +and functions between Python and Rust. + +The approach we are taking in the DataFusion in Python project is to incrementally expose +more portions of the DataFusion project via FFI interfaces. This allows users to write Rust +code that does **not** require the ``datafusion-python`` crate as a dependency, expose their +code in Python via PyO3, and have it interact with the DataFusion Python package. + +Early adopters of this approach include `delta-rs <https://delta-io.github.io/delta-rs/>`_ +who has adapted their Table Provider for use in ```datafusion-python``` with only a few lines +of code. Also, the DataFusion Python project uses the existing definitions from +`Apache Arrow CStream Interface <https://arrow.apache.org/docs/format/CStreamInterface.html>`_ +to support importing **and** exporting tables. Any Python package that supports reading +the Arrow C Stream interface can work with DataFusion Python out of the box! You can read +more about working with Arrow sources in the :ref:`Data Sources <user_guide_data_sources>` +page. + +To learn more about the Foreign Function Interface in Rust, the +`Rustonomicon <https://doc.rust-lang.org/nomicon/ffi.html>`_ is a good resource. + +Inspiration from Arrow +---------------------- + +DataFusion is built upon `Apache Arrow <https://arrow.apache.org/>`_. The canonical Python +Arrow implementation, `pyarrow <https://arrow.apache.org/docs/python/index.html>`_ provides +an excellent way to share Arrow data between Python projects without performing any copy +operations on the data. They do this by using a well defined set of interfaces. You can +find the details about their stream interface +`here <https://arrow.apache.org/docs/format/CStreamInterface.html>`_. The +`Rust Arrow Implementation <https://github.com/apache/arrow-rs>`_ also supports these +``C`` style definitions via the Foreign Function Interface. + +In addition to using these interfaces to transfer Arrow data between libraries, ``pyarrow`` +goes one step further to make sharing the interfaces easier in Python. They do this +by exposing PyCapsules that contain the expected functionality. + +You can learn more about PyCapsules from the official +`Python online documentation <https://docs.python.org/3/c-api/capsule.html>`_. PyCapsules +have excellent support in PyO3 already. The +`PyO3 online documentation <https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule>`_ is a good source +for more details on using PyCapsules in Rust. + +Two lessons we leverage from the Arrow project in DataFusion Python are: + +- We reuse the existing Arrow FFI functionality wherever possible. +- We expose PyCapsules that contain a FFI stable struct. + +Implementation Details +---------------------- + +The bulk of the code necessary to perform our FFI operations is in the upstream +`DataFusion <https://datafusion.apache.org/>`_ core repository. You can review the code and +documentation in the `datafusion-ffi`_ crate. + +Our FFI implementation is narrowly focused at sharing data and functions with Rust backed +libraries. This allows us to use the `abi_stable crate <https://crates.io/crates/abi_stable>`_. +This is an excellent crate that allows for easy conversion between Rust native types +and FFI-safe alternatives. For example, if you needed to pass a ``Vec<String>`` via FFI, +you can simply convert it to a ``RVec<RString>`` in an intuitive manner. It also supports +features like ``RResult`` and ``ROption`` that do not have an obvious translation to a +C equivalent. + +The `datafusion-ffi`_ crate has been designed to make it easy to convert from DataFusion +traits into their FFI counterparts. For example, if you have defined a custom +`TableProvider <https://docs.rs/datafusion/45.0.0/datafusion/catalog/trait.TableProvider.html>`_ +and you want to create a sharable FFI counterpart, you could write: + +.. code-block:: rust + + let my_provider = MyTableProvider::default(); + let ffi_provider = FFI_TableProvider::new(Arc::new(my_provider), false, None); + +If you were interfacing with a library that provided the above ``FFI_TableProvider`` and +you needed to turn it back into an ``TableProvider``, you can turn it into a +``ForeignTableProvider`` with implements the ``TableProvider`` trait. + +.. code-block:: rust + + let foreign_provider: ForeignTableProvider = ffi_provider.into(); + +If you review the code in `datafusion-ffi`_ you will find that each of the traits we share +across the boundary has two portions, one with a ``FFI_`` prefix and one with a ``Foreign`` +prefix. This is used to distinguish which side of the FFI boundary that struct is +designed to be used on. The structures with the ``FFI_`` prefix are to be used on the +**provider** of the structure. In the example we're showing, this means the code that has +written the underlying ``TableProvider`` implementation to access your custom data source. +The structures with the ``Foreign`` prefix are to be used by the receiver. In this case, +it is the ``datafusion-python`` library. + +In order to share these FFI structures, we need to wrap them in some kind of Python object +that can be used to interface from one package to another. As described in the above +section on our inspiration from Arrow, we use ``PyCapsule``. We can create a ``PyCapsule`` +for our provider thusly: + +.. code-block:: rust + + let name = CString::new("datafusion_table_provider")?; + let my_capsule = PyCapsule::new_bound(py, provider, Some(name))?; + +On the receiving side, turn this pycapsule object into the ``FFI_TableProvider``, which +can then be turned into a ``ForeignTableProvider`` the associated code is: + +.. code-block:: rust + + let capsule = capsule.downcast::<PyCapsule>()?; + let provider = unsafe { capsule.reference::<FFI_TableProvider>() }; + +By convention the ``datafusion-python`` library expects a Python object that has a +``TableProvider`` PyCapsule to have this capsule accessible by calling a function named +``__datafusion_table_provider__``. You can see a complete working example of how to +share a ``TableProvider`` from one python library to DataFusion Python in the +`repository examples folder <https://github.com/apache/datafusion-python/tree/main/examples/ffi-table-provider>`_. + +This section has been written using ``TableProvider`` as an example. It is the first +extension that has been written using this approach and the most thoroughly implemented. +As we continue to expose more of the DataFusion features, we intend to follow this same +design pattern. + +Alternative Approach +-------------------- + +Suppose you needed to expose some other features of DataFusion and you could not wait +for the upstream repository to implement the FFI approach we describe. In this case +you decide to create your dependency on the ``datafusion-python`` crate instead. + +As we discussed, this is not guaranteed to work across different compiler versions and +optimization levels. If you wish to go down this route, there are two approaches we +have identified you can use. + +#. Re-export all of ``datafusion-python`` yourself with your extensions built in. +#. Carefully synchonize your software releases with the ``datafusion-python`` CI build + system so that your libraries use the exact same compiler, features, and + optimization level. + +We currently do not recommend either of these approaches as they are difficult to +maintain over a long period. Additionally, they require a tight version coupling +between libraries. + +Status of Work +-------------- + +At the time of this writing, the FFI features are under active development. To see +the latest status, we recommend reviewing the code in the `datafusion-ffi`_ crate. + +.. _datafusion-ffi: https://crates.io/crates/datafusion-ffi
diff --git a/_sources/contributor-guide/introduction.rst.txt b/_sources/contributor-guide/introduction.rst.txt index 25f2c21..2fba641 100644 --- a/_sources/contributor-guide/introduction.rst.txt +++ b/_sources/contributor-guide/introduction.rst.txt
@@ -118,7 +118,7 @@ .. code-block:: implementation=CPython - version=3.8 + version=3.9 shared=true abi3=true lib_name=python3.12
diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 34eb23b..558b2d5 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt
@@ -85,6 +85,7 @@ :caption: CONTRIBUTOR GUIDE contributor-guide/introduction + contributor-guide/ffi .. _toc.api: .. toctree::
diff --git a/_sources/user-guide/common-operations/index.rst.txt b/_sources/user-guide/common-operations/index.rst.txt index d7c708c..7abd1f1 100644 --- a/_sources/user-guide/common-operations/index.rst.txt +++ b/_sources/user-guide/common-operations/index.rst.txt
@@ -23,6 +23,7 @@ .. toctree:: :maxdepth: 2 + views basic-info select-and-filter expressions
diff --git a/_sources/user-guide/common-operations/views.rst.txt b/_sources/user-guide/common-operations/views.rst.txt new file mode 100644 index 0000000..df11e3a --- /dev/null +++ b/_sources/user-guide/common-operations/views.rst.txt
@@ -0,0 +1,58 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +====================== +Registering Views +====================== + +You can use the context's ``register_view`` method to register a DataFrame as a view + +.. code-block:: python + + from datafusion import SessionContext, col, literal + + # Create a DataFusion context + ctx = SessionContext() + + # Create sample data + data = {"a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50]} + + # Create a DataFrame from the dictionary + df = ctx.from_pydict(data, "my_table") + + # Filter the DataFrame (for example, keep rows where a > 2) + df_filtered = df.filter(col("a") > literal(2)) + + # Register the dataframe as a view with the context + ctx.register_view("view1", df_filtered) + + # Now run a SQL query against the registered view + df_view = ctx.sql("SELECT * FROM view1") + + # Collect the results + results = df_view.collect() + + # Convert results to a list of dictionaries for display + result_dicts = [batch.to_pydict() for batch in results] + + print(result_dicts) + +This will output: + +.. code-block:: python + + [{'a': [3, 4, 5], 'b': [30, 40, 50]}]
diff --git a/autoapi/datafusion/catalog/index.html b/autoapi/datafusion/catalog/index.html index 3f250b3..d79211a 100644 --- a/autoapi/datafusion/catalog/index.html +++ b/autoapi/datafusion/catalog/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -456,19 +471,19 @@ </a> <ul class="nav section-nav flex-column"> <li class="toc-h4 nav-item toc-entry"> - <a class="reference internal nav-link" href="#datafusion.catalog.Table.schema"> + <a class="reference internal nav-link" href="#datafusion.catalog.Table.kind"> <code class="docutils literal notranslate"> <span class="pre"> - Table.schema() + Table.kind </span> </code> </a> </li> <li class="toc-h4 nav-item toc-entry"> - <a class="reference internal nav-link" href="#datafusion.catalog.Table.kind"> + <a class="reference internal nav-link" href="#datafusion.catalog.Table.schema"> <code class="docutils literal notranslate"> <span class="pre"> - Table.kind + Table.schema </span> </code> </a> @@ -581,18 +596,18 @@ <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.catalog.</span></span><span class="sig-name descname"><span class="pre">Table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.Table</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.catalog.Table" title="Link to this definition">¶</a></dt> <dd><p>DataFusion table.</p> <p>This constructor is not typically called by the end user.</p> -<dl class="py method"> -<dt class="sig sig-object py" id="datafusion.catalog.Table.schema"> -<span class="sig-name descname"><span class="pre">schema</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Schema</span></span></span><a class="headerlink" href="#datafusion.catalog.Table.schema" title="Link to this definition">¶</a></dt> -<dd><p>Returns the schema associated with this table.</p> -</dd></dl> - <dl class="py property"> <dt class="sig sig-object py" id="datafusion.catalog.Table.kind"> <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">kind</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#datafusion.catalog.Table.kind" title="Link to this definition">¶</a></dt> <dd><p>Returns the kind of table.</p> </dd></dl> +<dl class="py property"> +<dt class="sig sig-object py" id="datafusion.catalog.Table.schema"> +<em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">schema</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">pyarrow.Schema</span></em><a class="headerlink" href="#datafusion.catalog.Table.schema" title="Link to this definition">¶</a></dt> +<dd><p>Returns the schema associated with this table.</p> +</dd></dl> + <dl class="py attribute"> <dt class="sig sig-object py" id="datafusion.catalog.Table.table"> <span class="sig-name descname"><span class="pre">table</span></span><a class="headerlink" href="#datafusion.catalog.Table.table" title="Link to this definition">¶</a></dt>
diff --git a/autoapi/datafusion/context/index.html b/autoapi/datafusion/context/index.html index e2026ed..7715d6a 100644 --- a/autoapi/datafusion/context/index.html +++ b/autoapi/datafusion/context/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -813,6 +828,15 @@ </a> </li> <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.context.SessionContext.global_ctx"> + <code class="docutils literal notranslate"> + <span class="pre"> + SessionContext.global_ctx() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.context.SessionContext.read_avro"> <code class="docutils literal notranslate"> <span class="pre"> @@ -975,6 +999,15 @@ </a> </li> <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_view"> + <code class="docutils literal notranslate"> + <span class="pre"> + SessionContext.register_view() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.context.SessionContext.session_id"> <code class="docutils literal notranslate"> <span class="pre"> @@ -1595,7 +1628,7 @@ <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>partitions</strong> – <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code> partitions to register.</p></li> +<li><p><strong>partitions</strong> – <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code> partitions to register.</p></li> <li><p><strong>name</strong> – Resultant dataframe name.</p></li> <li><p><strong>schema</strong> – Schema for the partitions.</p></li> </ul> @@ -1746,6 +1779,17 @@ </dd></dl> <dl class="py method"> +<dt class="sig sig-object py" id="datafusion.context.SessionContext.global_ctx"> +<em class="property"><span class="pre">classmethod</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">global_ctx</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">SessionContext</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.global_ctx" title="Link to this definition">¶</a></dt> +<dd><p>Retrieve the global context as a <cite>SessionContext</cite> wrapper.</p> +<dl class="field-list simple"> +<dt class="field-odd">Returns<span class="colon">:</span></dt> +<dd class="field-odd"><p>A <cite>SessionContext</cite> object that wraps the global <cite>SessionContextInternal</cite>.</p> +</dd> +</dl> +</dd></dl> + +<dl class="py method"> <dt class="sig sig-object py" id="datafusion.context.SessionContext.read_avro"> <span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.avro'</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.read_avro" title="Link to this definition">¶</a></dt> <dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p> @@ -1817,7 +1861,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.context.SessionContext.read_parquet"> -<span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.read_parquet" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="../expr/index.html#datafusion.expr.SortExpr" title="datafusion.expr.SortExpr"><span class="pre">datafusion.expr.SortExpr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.read_parquet" title="Link to this definition">¶</a></dt> <dd><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> @@ -1900,7 +1944,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.context.SessionContext.register_dataset"> <span class="sig-name descname"><span class="pre">register_dataset</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.dataset.Dataset</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.register_dataset" title="Link to this definition">¶</a></dt> -<dd><p>Register a <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.dataset.Dataset</span></code> as a table.</p> +<dd><p>Register a <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.dataset.Dataset</span></code> as a table.</p> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> @@ -1972,7 +2016,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.context.SessionContext.register_parquet"> -<span class="sig-name descname"><span class="pre">register_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.register_parquet" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">register_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.SortExpr" title="datafusion.expr.SortExpr"><span class="pre">datafusion.expr.SortExpr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.register_parquet" title="Link to this definition">¶</a></dt> <dd><p>Register a Parquet file as a table.</p> <p>The registered table can be referenced from SQL statement executed against this context.</p> @@ -2054,6 +2098,20 @@ </dd></dl> <dl class="py method"> +<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_view"> +<span class="sig-name descname"><span class="pre">register_view</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">df</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.register_view" title="Link to this definition">¶</a></dt> +<dd><p>Register a :py:class: <cite>~datafusion.detaframe.DataFrame</cite> as a view.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>name</strong> (<em>str</em>) – The name to register the view under.</p></li> +<li><p><strong>df</strong> (<a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><em>DataFrame</em></a>) – The DataFrame to be converted into a view and registered.</p></li> +</ul> +</dd> +</dl> +</dd></dl> + +<dl class="py method"> <dt class="sig sig-object py" id="datafusion.context.SessionContext.session_id"> <span class="sig-name descname"><span class="pre">session_id</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.session_id" title="Link to this definition">¶</a></dt> <dd><p>Return an id that uniquely identifies this <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>.</p>
diff --git a/autoapi/datafusion/dataframe/index.html b/autoapi/datafusion/dataframe/index.html index da5003d..2ed6116 100644 --- a/autoapi/datafusion/dataframe/index.html +++ b/autoapi/datafusion/dataframe/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -661,6 +676,15 @@ </a> </li> <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.into_view"> + <code class="docutils literal notranslate"> + <span class="pre"> + DataFrame.into_view() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.join"> <code class="docutils literal notranslate"> <span class="pre"> @@ -1063,7 +1087,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__getitem__"> -<span class="sig-name descname"><span class="pre">__getitem__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__getitem__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__getitem__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__getitem__" title="Link to this definition">¶</a></dt> <dd><p>Return a new :py:class`DataFrame` with the specified column or columns.</p> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> @@ -1330,6 +1354,12 @@ </dd></dl> <dl class="py method"> +<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.into_view"> +<span class="sig-name descname"><span class="pre">into_view</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Table</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.into_view" title="Link to this definition">¶</a></dt> +<dd><p>Convert DataFrame as a ViewTable which can be used in register_table.</p> +</dd></dl> + +<dl class="py method"> <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.join"> <span class="sig-name descname"><span class="pre">join</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Sequence</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">how</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'inner'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'full'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'semi'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'anti'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'inner'</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">left_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">right_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">join_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.join" title="Link to this definition">¶</a></dt> <dt class="sig sig-object py">
diff --git a/autoapi/datafusion/expr/index.html b/autoapi/datafusion/expr/index.html index 78ecdc6..ae12073 100644 --- a/autoapi/datafusion/expr/index.html +++ b/autoapi/datafusion/expr/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -1898,7 +1913,7 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.expr.Expr"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.expr.</span></span><span class="sig-name descname"><span class="pre">Expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.expr.Expr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.expr.Expr" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.expr.</span></span><span class="sig-name descname"><span class="pre">Expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.expr.RawExpr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.expr.Expr" title="Link to this definition">¶</a></dt> <dd><p>Expression object.</p> <p>Expressions are one of the core concepts in DataFusion. See <a class="reference internal" href="../../../user-guide/common-operations/expressions.html#expressions"><span class="std std-ref">Expressions</span></a> in the online documentation for more information.</p> @@ -1918,7 +1933,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.expr.Expr.__eq__"> -<span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.__eq__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.__eq__" title="Link to this definition">¶</a></dt> <dd><p>Equal to.</p> <p>Accepts either an expression or any valid PyArrow scalar literal value.</p> </dd></dl> @@ -1982,7 +1997,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.expr.Expr.__ne__"> -<span class="sig-name descname"><span class="pre">__ne__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.__ne__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__ne__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.__ne__" title="Link to this definition">¶</a></dt> <dd><p>Not equal to.</p> <p>Accepts either an expression or any valid PyArrow scalar literal value.</p> </dd></dl> @@ -2048,7 +2063,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.expr.Expr.cast"> -<span class="sig-name descname"><span class="pre">cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">to</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">float</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">bool</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.cast" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">to</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">type</span><span class="p"><span class="pre">[</span></span><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">bool</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.expr.Expr.cast" title="Link to this definition">¶</a></dt> <dd><p>Cast to a new data type.</p> </dd></dl> @@ -2301,7 +2316,7 @@ <dl class="py attribute"> <dt class="sig sig-object py" id="datafusion.expr.Expr._to_pyarrow_types"> -<span class="sig-name descname"><span class="pre">_to_pyarrow_types</span></span><a class="headerlink" href="#datafusion.expr.Expr._to_pyarrow_types" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">_to_pyarrow_types</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">ClassVar</span><span class="p"><span class="pre">[</span></span><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">type</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#datafusion.expr.Expr._to_pyarrow_types" title="Link to this definition">¶</a></dt> <dd></dd></dl> <dl class="py attribute"> @@ -2416,7 +2431,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.expr.WindowFrame.get_upper_bound"> -<span class="sig-name descname"><span class="pre">get_upper_bound</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.expr.WindowFrame.get_upper_bound" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">get_upper_bound</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.expr.WindowFrameBound" title="datafusion.expr.WindowFrameBound"><span class="pre">WindowFrameBound</span></a></span></span><a class="headerlink" href="#datafusion.expr.WindowFrame.get_upper_bound" title="Link to this definition">¶</a></dt> <dd><p>Returns end bound.</p> </dd></dl>
diff --git a/autoapi/datafusion/functions/index.html b/autoapi/datafusion/functions/index.html index 51bc53b..05ced22 100644 --- a/autoapi/datafusion/functions/index.html +++ b/autoapi/datafusion/functions/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -1991,6 +2006,15 @@ </a> </li> <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.functions.regexp_count"> + <code class="docutils literal notranslate"> + <span class="pre"> + regexp_count() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.functions.regexp_like"> <code class="docutils literal notranslate"> <span class="pre"> @@ -2405,6 +2429,15 @@ </a> </li> <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.functions.to_timestamp_nanos"> + <code class="docutils literal notranslate"> + <span class="pre"> + to_timestamp_nanos() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.functions.to_timestamp_seconds"> <code class="docutils literal notranslate"> <span class="pre"> @@ -3091,144 +3124,150 @@ <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.rank" title="datafusion.functions.rank"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rank</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Create a rank window function.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regexp_like" title="datafusion.functions.regexp_like"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_like</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regexp_count" title="datafusion.functions.regexp_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_count</span></code></a>(→ datafusion.expr.Expr)</p></td> +<td><p>Returns the number of matches in a string.</p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regexp_like" title="datafusion.functions.regexp_like"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_like</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Find if any regular expression (regex) matches exist.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regexp_match" title="datafusion.functions.regexp_match"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_match</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regexp_match" title="datafusion.functions.regexp_match"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_match</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Perform regular expression (regex) matching.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regexp_replace" title="datafusion.functions.regexp_replace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_replace</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regexp_replace" title="datafusion.functions.regexp_replace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regexp_replace</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Replaces substring(s) matching a PCRE-like regular expression.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_avgx" title="datafusion.functions.regr_avgx"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_avgx</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_avgx" title="datafusion.functions.regr_avgx"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_avgx</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the average of the independent variable <code class="docutils literal notranslate"><span class="pre">x</span></code>.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_avgy" title="datafusion.functions.regr_avgy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_avgy</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_avgy" title="datafusion.functions.regr_avgy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_avgy</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the average of the dependent variable <code class="docutils literal notranslate"><span class="pre">y</span></code>.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_count" title="datafusion.functions.regr_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_count</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_count" title="datafusion.functions.regr_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_count</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Counts the number of rows in which both expressions are not null.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_intercept" title="datafusion.functions.regr_intercept"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_intercept</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_intercept" title="datafusion.functions.regr_intercept"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_intercept</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the intercept from the linear regression.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_r2" title="datafusion.functions.regr_r2"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_r2</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_r2" title="datafusion.functions.regr_r2"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_r2</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the R-squared value from linear regression.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_slope" title="datafusion.functions.regr_slope"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_slope</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_slope" title="datafusion.functions.regr_slope"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_slope</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the slope from linear regression.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_sxx" title="datafusion.functions.regr_sxx"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_sxx</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_sxx" title="datafusion.functions.regr_sxx"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_sxx</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the sum of squares of the independent variable <code class="docutils literal notranslate"><span class="pre">x</span></code>.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_sxy" title="datafusion.functions.regr_sxy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_sxy</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_sxy" title="datafusion.functions.regr_sxy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_sxy</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the sum of products of pairs of numbers.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.regr_syy" title="datafusion.functions.regr_syy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_syy</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.regr_syy" title="datafusion.functions.regr_syy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">regr_syy</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the sum of squares of the dependent variable <code class="docutils literal notranslate"><span class="pre">y</span></code>.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.repeat" title="datafusion.functions.repeat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">repeat</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.repeat" title="datafusion.functions.repeat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">repeat</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Repeats the <code class="docutils literal notranslate"><span class="pre">string</span></code> to <code class="docutils literal notranslate"><span class="pre">n</span></code> times.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.replace" title="datafusion.functions.replace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">replace</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.replace" title="datafusion.functions.replace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">replace</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Replaces all occurrences of <code class="docutils literal notranslate"><span class="pre">from_val</span></code> with <code class="docutils literal notranslate"><span class="pre">to_val</span></code> in the <code class="docutils literal notranslate"><span class="pre">string</span></code>.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.reverse" title="datafusion.functions.reverse"><code class="xref py py-obj docutils literal notranslate"><span class="pre">reverse</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.reverse" title="datafusion.functions.reverse"><code class="xref py py-obj docutils literal notranslate"><span class="pre">reverse</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Reverse the string argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.right" title="datafusion.functions.right"><code class="xref py py-obj docutils literal notranslate"><span class="pre">right</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.right" title="datafusion.functions.right"><code class="xref py py-obj docutils literal notranslate"><span class="pre">right</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the last <code class="docutils literal notranslate"><span class="pre">n</span></code> characters in the <code class="docutils literal notranslate"><span class="pre">string</span></code>.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.round" title="datafusion.functions.round"><code class="xref py py-obj docutils literal notranslate"><span class="pre">round</span></code></a>() → datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.round" title="datafusion.functions.round"><code class="xref py py-obj docutils literal notranslate"><span class="pre">round</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Round the argument to the nearest integer.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.row_number" title="datafusion.functions.row_number"><code class="xref py py-obj docutils literal notranslate"><span class="pre">row_number</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.row_number" title="datafusion.functions.row_number"><code class="xref py py-obj docutils literal notranslate"><span class="pre">row_number</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Create a row number window function.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.rpad" title="datafusion.functions.rpad"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rpad</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.rpad" title="datafusion.functions.rpad"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rpad</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Add right padding to a string.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.rtrim" title="datafusion.functions.rtrim"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rtrim</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.rtrim" title="datafusion.functions.rtrim"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rtrim</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Removes all characters, spaces by default, from the end of a string.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sha224" title="datafusion.functions.sha224"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha224</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sha224" title="datafusion.functions.sha224"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha224</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the SHA-224 hash of a binary string.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sha256" title="datafusion.functions.sha256"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha256</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sha256" title="datafusion.functions.sha256"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha256</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the SHA-256 hash of a binary string.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sha384" title="datafusion.functions.sha384"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha384</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sha384" title="datafusion.functions.sha384"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha384</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the SHA-384 hash of a binary string.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sha512" title="datafusion.functions.sha512"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha512</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sha512" title="datafusion.functions.sha512"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha512</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the SHA-512 hash of a binary string.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.signum" title="datafusion.functions.signum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">signum</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.signum" title="datafusion.functions.signum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">signum</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the sign of the argument (-1, 0, +1).</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sin" title="datafusion.functions.sin"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sin</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sin" title="datafusion.functions.sin"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sin</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the sine of the argument.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sinh" title="datafusion.functions.sinh"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sinh</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sinh" title="datafusion.functions.sinh"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sinh</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the hyperbolic sine of the argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.split_part" title="datafusion.functions.split_part"><code class="xref py py-obj docutils literal notranslate"><span class="pre">split_part</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.split_part" title="datafusion.functions.split_part"><code class="xref py py-obj docutils literal notranslate"><span class="pre">split_part</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Split a string and return one part.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sqrt" title="datafusion.functions.sqrt"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sqrt</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sqrt" title="datafusion.functions.sqrt"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sqrt</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the square root of the argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.starts_with" title="datafusion.functions.starts_with"><code class="xref py py-obj docutils literal notranslate"><span class="pre">starts_with</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.starts_with" title="datafusion.functions.starts_with"><code class="xref py py-obj docutils literal notranslate"><span class="pre">starts_with</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns true if string starts with prefix.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.stddev" title="datafusion.functions.stddev"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.stddev" title="datafusion.functions.stddev"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the standard deviation of the argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.stddev_pop" title="datafusion.functions.stddev_pop"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev_pop</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.stddev_pop" title="datafusion.functions.stddev_pop"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev_pop</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the population standard deviation of the argument.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.stddev_samp" title="datafusion.functions.stddev_samp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev_samp</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.stddev_samp" title="datafusion.functions.stddev_samp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">stddev_samp</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the sample standard deviation of the argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.string_agg" title="datafusion.functions.string_agg"><code class="xref py py-obj docutils literal notranslate"><span class="pre">string_agg</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.string_agg" title="datafusion.functions.string_agg"><code class="xref py py-obj docutils literal notranslate"><span class="pre">string_agg</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Concatenates the input strings.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.strpos" title="datafusion.functions.strpos"><code class="xref py py-obj docutils literal notranslate"><span class="pre">strpos</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.strpos" title="datafusion.functions.strpos"><code class="xref py py-obj docutils literal notranslate"><span class="pre">strpos</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Finds the position from where the <code class="docutils literal notranslate"><span class="pre">substring</span></code> matches the <code class="docutils literal notranslate"><span class="pre">string</span></code>.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.struct" title="datafusion.functions.struct"><code class="xref py py-obj docutils literal notranslate"><span class="pre">struct</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.struct" title="datafusion.functions.struct"><code class="xref py py-obj docutils literal notranslate"><span class="pre">struct</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns a struct with the given arguments.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.substr" title="datafusion.functions.substr"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substr</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.substr" title="datafusion.functions.substr"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substr</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Substring from the <code class="docutils literal notranslate"><span class="pre">position</span></code> to the end.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.substr_index" title="datafusion.functions.substr_index"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substr_index</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.substr_index" title="datafusion.functions.substr_index"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substr_index</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns an indexed substring.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.substring" title="datafusion.functions.substring"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substring</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.substring" title="datafusion.functions.substring"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substring</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Substring from the <code class="docutils literal notranslate"><span class="pre">position</span></code> with <code class="docutils literal notranslate"><span class="pre">length</span></code> characters.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.sum" title="datafusion.functions.sum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sum</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.sum" title="datafusion.functions.sum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sum</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Computes the sum of a set of numbers.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.tan" title="datafusion.functions.tan"><code class="xref py py-obj docutils literal notranslate"><span class="pre">tan</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.tan" title="datafusion.functions.tan"><code class="xref py py-obj docutils literal notranslate"><span class="pre">tan</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the tangent of the argument.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.tanh" title="datafusion.functions.tanh"><code class="xref py py-obj docutils literal notranslate"><span class="pre">tanh</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.tanh" title="datafusion.functions.tanh"><code class="xref py py-obj docutils literal notranslate"><span class="pre">tanh</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Returns the hyperbolic tangent of the argument.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.to_hex" title="datafusion.functions.to_hex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_hex</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.to_hex" title="datafusion.functions.to_hex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_hex</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Converts an integer to a hexadecimal string.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp" title="datafusion.functions.to_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp" title="datafusion.functions.to_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in nanoseconds.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_micros" title="datafusion.functions.to_timestamp_micros"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_micros</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_micros" title="datafusion.functions.to_timestamp_micros"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_micros</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in microseconds.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_millis" title="datafusion.functions.to_timestamp_millis"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_millis</span></code></a>(→ datafusion.expr.Expr)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_millis" title="datafusion.functions.to_timestamp_millis"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_millis</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in milliseconds.</p></td> </tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_nanos" title="datafusion.functions.to_timestamp_nanos"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_nanos</span></code></a>(→ datafusion.expr.Expr)</p></td> +<td><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in nanoseconds.</p></td> +</tr> <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.to_timestamp_seconds" title="datafusion.functions.to_timestamp_seconds"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_timestamp_seconds</span></code></a>(→ datafusion.expr.Expr)</p></td> <td><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in seconds.</p></td> </tr> @@ -4083,7 +4122,7 @@ <dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.decode"> -<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.decode" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.decode" title="Link to this definition">¶</a></dt> <dd><p>Decode the <code class="docutils literal notranslate"><span class="pre">input</span></code>, using the <code class="docutils literal notranslate"><span class="pre">encoding</span></code>. encoding can be base64 or hex.</p> </dd></dl> @@ -4136,7 +4175,7 @@ <dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.encode"> -<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">encode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.encode" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">encode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.encode" title="Link to this definition">¶</a></dt> <dd><p>Encode the <code class="docutils literal notranslate"><span class="pre">input</span></code>, using the <code class="docutils literal notranslate"><span class="pre">encoding</span></code>. encoding can be base64 or hex.</p> </dd></dl> @@ -4895,6 +4934,14 @@ </dd></dl> <dl class="py function"> +<dt class="sig sig-object py" id="datafusion.functions.regexp_count"> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">regexp_count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pattern</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">flags</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.regexp_count" title="Link to this definition">¶</a></dt> +<dd><p>Returns the number of matches in a string.</p> +<p>Optional start position (the first position is 1) to search for the regular +expression.</p> +</dd></dl> + +<dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.regexp_like"> <span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">regexp_like</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">regex</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">flags</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.regexp_like" title="Link to this definition">¶</a></dt> <dd><p>Find if any regular expression (regex) matches exist.</p> @@ -5117,7 +5164,7 @@ <dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.round"> -<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">round</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">decimal_places</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">Expr.literal(0)</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.round" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">round</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">decimal_places</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.round" title="Link to this definition">¶</a></dt> <dd><p>Round the argument to the nearest integer.</p> <p>If the optional <code class="docutils literal notranslate"><span class="pre">decimal_places</span></code> is specified, round to the nearest number of decimal places. You can specify a negative number of decimal places. For example @@ -5376,6 +5423,13 @@ </dd></dl> <dl class="py function"> +<dt class="sig sig-object py" id="datafusion.functions.to_timestamp_nanos"> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">to_timestamp_nanos</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">formatters</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.to_timestamp_nanos" title="Link to this definition">¶</a></dt> +<dd><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in nanoseconds.</p> +<p>See <a class="reference internal" href="#datafusion.functions.to_timestamp" title="datafusion.functions.to_timestamp"><code class="xref py py-func docutils literal notranslate"><span class="pre">to_timestamp()</span></code></a> for a description on how to use formatters.</p> +</dd></dl> + +<dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.to_timestamp_seconds"> <span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">to_timestamp_seconds</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">formatters</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.to_timestamp_seconds" title="Link to this definition">¶</a></dt> <dd><p>Converts a string and optional formats to a <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> in seconds.</p> @@ -5414,7 +5468,7 @@ <dl class="py function"> <dt class="sig sig-object py" id="datafusion.functions.uuid"> -<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">uuid</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.uuid" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">uuid</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.uuid" title="Link to this definition">¶</a></dt> <dd><p>Returns uuid v4 as a string value.</p> </dd></dl>
diff --git a/autoapi/datafusion/index.html b/autoapi/datafusion/index.html index f4c3d35..7e5257e 100644 --- a/autoapi/datafusion/index.html +++ b/autoapi/datafusion/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="object_store/index.html"> datafusion.object_store </a> @@ -1645,19 +1660,19 @@ </a> <ul class="nav section-nav flex-column"> <li class="toc-h4 nav-item toc-entry"> - <a class="reference internal nav-link" href="#datafusion.Table.schema"> + <a class="reference internal nav-link" href="#datafusion.Table.kind"> <code class="docutils literal notranslate"> <span class="pre"> - Table.schema() + Table.kind </span> </code> </a> </li> <li class="toc-h4 nav-item toc-entry"> - <a class="reference internal nav-link" href="#datafusion.Table.kind"> + <a class="reference internal nav-link" href="#datafusion.Table.schema"> <code class="docutils literal notranslate"> <span class="pre"> - Table.kind + Table.schema </span> </code> </a> @@ -1739,6 +1754,42 @@ </a> </li> <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.WindowUDF._create_window_udf"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._create_window_udf() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.WindowUDF._create_window_udf_decorator"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._create_window_udf_decorator() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.WindowUDF._get_default_name"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._get_default_name() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.WindowUDF._normalize_input_types"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._normalize_input_types() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.WindowUDF.udwf"> <code class="docutils literal notranslate"> <span class="pre"> @@ -1795,6 +1846,42 @@ </a> </li> <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.read_avro"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_avro() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.read_csv"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_csv() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.read_json"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_json() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.read_parquet"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_parquet() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.DFSchema"> <code class="docutils literal notranslate"> <span class="pre"> @@ -1803,6 +1890,33 @@ </code> </a> </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udaf"> + <code class="docutils literal notranslate"> + <span class="pre"> + udaf + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf"> + <code class="docutils literal notranslate"> + <span class="pre"> + udf + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udwf"> + <code class="docutils literal notranslate"> + <span class="pre"> + udwf + </span> + </code> + </a> + </li> </ul> </li> </ul> @@ -1841,6 +1955,7 @@ <li class="toctree-l1"><a class="reference internal" href="expr/index.html">datafusion.expr</a></li> <li class="toctree-l1"><a class="reference internal" href="functions/index.html">datafusion.functions</a></li> <li class="toctree-l1"><a class="reference internal" href="input/index.html">datafusion.input</a></li> +<li class="toctree-l1"><a class="reference internal" href="io/index.html">datafusion.io</a></li> <li class="toctree-l1"><a class="reference internal" href="object_store/index.html">datafusion.object_store</a></li> <li class="toctree-l1"><a class="reference internal" href="plan/index.html">datafusion.plan</a></li> <li class="toctree-l1"><a class="reference internal" href="record_batch/index.html">datafusion.record_batch</a></li> @@ -1856,6 +1971,15 @@ <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.DFSchema" title="datafusion.DFSchema"><code class="xref py py-obj docutils literal notranslate"><span class="pre">DFSchema</span></code></a></p></td> <td><p></p></td> </tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.udaf" title="datafusion.udaf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udaf</span></code></a></p></td> +<td><p></p></td> +</tr> +<tr class="row-odd"><td><p><a class="reference internal" href="udf/index.html#module-datafusion.udf" title="datafusion.udf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udf</span></code></a></p></td> +<td><p></p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.udwf" title="datafusion.udwf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udwf</span></code></a></p></td> +<td><p></p></td> +</tr> </tbody> </table> </section> @@ -1885,7 +2009,7 @@ <td><p>Logical Plan.</p></td> </tr> <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.RecordBatch" title="datafusion.RecordBatch"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RecordBatch</span></code></a></p></td> -<td><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p></td> +<td><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p></td> </tr> <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.RecordBatchStream" title="datafusion.RecordBatchStream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RecordBatchStream</span></code></a></p></td> <td><p>This class represents a stream of record batches.</p></td> @@ -1918,18 +2042,30 @@ <h2>Functions<a class="headerlink" href="#functions" title="Link to this heading">¶</a></h2> <table class="autosummary longtable table autosummary"> <tbody> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.col" title="datafusion.col"><code class="xref py py-obj docutils literal notranslate"><span class="pre">col</span></code></a>(value)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.col" title="datafusion.col"><code class="xref py py-obj docutils literal notranslate"><span class="pre">col</span></code></a>(→ expr.Expr)</p></td> <td><p>Create a column expression.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.column" title="datafusion.column"><code class="xref py py-obj docutils literal notranslate"><span class="pre">column</span></code></a>(value)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.column" title="datafusion.column"><code class="xref py py-obj docutils literal notranslate"><span class="pre">column</span></code></a>(→ expr.Expr)</p></td> <td><p>Create a column expression.</p></td> </tr> -<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.lit" title="datafusion.lit"><code class="xref py py-obj docutils literal notranslate"><span class="pre">lit</span></code></a>(value)</p></td> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.lit" title="datafusion.lit"><code class="xref py py-obj docutils literal notranslate"><span class="pre">lit</span></code></a>(→ expr.Expr)</p></td> <td><p>Create a literal expression.</p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.literal" title="datafusion.literal"><code class="xref py py-obj docutils literal notranslate"><span class="pre">literal</span></code></a>(value)</p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.literal" title="datafusion.literal"><code class="xref py py-obj docutils literal notranslate"><span class="pre">literal</span></code></a>(→ expr.Expr)</p></td> <td><p>Create a literal expression.</p></td> </tr> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.read_avro" title="datafusion.read_avro"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_avro</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.read_csv" title="datafusion.read_csv"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_csv</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a CSV data source.</p></td> +</tr> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.read_json" title="datafusion.read_json"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_json</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a line-delimited JSON data source.</p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.read_parquet" title="datafusion.read_parquet"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_parquet</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p></td> +</tr> </tbody> </table> </section> @@ -1947,13 +2083,13 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.Accumulator.merge"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">merge</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">states</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Array</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.Accumulator.merge" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">merge</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">states</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Array</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.Accumulator.merge" title="Link to this definition">¶</a></dt> <dd><p>Merge a set of states.</p> </dd></dl> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.Accumulator.state"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">state</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Scalar</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.Accumulator.state" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">state</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Scalar</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.Accumulator.state" title="Link to this definition">¶</a></dt> <dd><p>Return the current state.</p> </dd></dl> @@ -1967,12 +2103,12 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.AggregateUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">AggregateUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">accumulator</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.AggregateUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">AggregateUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">accumulator</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.AggregateUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing scalar user-defined functions (UDF).</p> <p>Aggregate UDFs operate on a group of rows and return a single value. See also <a class="reference internal" href="#datafusion.ScalarUDF" title="datafusion.ScalarUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">ScalarUDF</span></code></a> for operating on a row by row basis.</p> <p>Instantiate a user-defined aggregate function (UDAF).</p> -<p>See <a class="reference internal" href="#datafusion.AggregateUDF.udaf" title="datafusion.AggregateUDF.udaf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udaf()</span></code></a> for a convenience function and argument +<p>See <a class="reference internal" href="#datafusion.udaf" title="datafusion.udaf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udaf()</span></code></a> for a convenience function and argument descriptions.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.AggregateUDF.__call__"> @@ -1984,44 +2120,78 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.AggregateUDF.udaf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">accum</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.AggregateUDF" title="datafusion.AggregateUDF"><span class="pre">AggregateUDF</span></a></span></span><a class="headerlink" href="#datafusion.AggregateUDF.udaf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Aggregate Function.</p> -<p>If your <a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><code class="xref py py-class docutils literal notranslate"><span class="pre">Accumulator</span></code></a> can be instantiated with no arguments, you -can simply pass it’s type as <code class="docutils literal notranslate"><span class="pre">accum</span></code>. If you need to pass additional arguments -to it’s constructor, you can define a lambda or a factory method. During runtime -the <a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><code class="xref py py-class docutils literal notranslate"><span class="pre">Accumulator</span></code></a> will be constructed for every instance in -which this UDAF is used. The following examples are all valid.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> -<span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.compute</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pc</span> - -<span class="k">class</span><span class="w"> </span><span class="nc">Summarize</span><span class="p">(</span><span class="n">Accumulator</span><span class="p">):</span> - <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bias</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">bias</span><span class="p">)</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Scalar</span><span class="p">]:</span> - <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="p">]</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">values</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> <span class="o">+</span> <span class="n">pc</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">values</span><span class="p">)</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">states</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> <span class="o">+</span> <span class="n">pc</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">states</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">evaluate</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Scalar</span><span class="p">:</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> - -<span class="k">def</span><span class="w"> </span><span class="nf">sum_bias_10</span><span class="p">()</span> <span class="o">-></span> <span class="n">Summarize</span><span class="p">:</span> - <span class="k">return</span> <span class="n">Summarize</span><span class="p">(</span><span class="mf">10.0</span><span class="p">)</span> - -<span class="n">udaf1</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="n">Summarize</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udaf2</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="n">sum_bias_10</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udaf3</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="k">lambda</span><span class="p">:</span> <span class="n">Summarize</span><span class="p">(</span><span class="mf">20.0</span><span class="p">),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -</pre></div> -</div> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.AggregateUDF" title="datafusion.AggregateUDF"><span class="pre">AggregateUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.AggregateUDF.udaf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">accum</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.Accumulator" title="datafusion.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.AggregateUDF" title="datafusion.AggregateUDF"><span class="pre">AggregateUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Aggregate Function (UDAF).</p> +<p>This class allows you to define an <strong>aggregate function</strong> that can be used in +data aggregation or window function calls.</p> +<dl> +<dt>Usage:</dt><dd><ul class="simple"> +<li><dl class="simple"> +<dt><strong>As a function</strong>: Call <a href="#id1"><span class="problematic" id="id2">`</span></a>udaf(accum, input_types, return_type, state_type,</dt><dd><p>volatility, name)`.</p> +</dd> +</dl> +</li> +<li><dl class="simple"> +<dt><strong>As a decorator</strong>: Use <a href="#id3"><span class="problematic" id="id4">`</span></a>@udaf(input_types, return_type, state_type,</dt><dd><p>volatility, name)`.</p> +</dd> +</dl> +</li> +</ul> +<p>When using <cite>udaf</cite> as a decorator, <strong>do not pass `accum` explicitly</strong>.</p> +</dd> +</dl> +<p><strong>Function example:</strong></p> +<blockquote> +<div><p>If your <cite>:py:class:Accumulator</cite> can be instantiated with no arguments, you +can simply pass it’s type as <cite>accum</cite>. If you need to pass additional +arguments to it’s constructor, you can define a lambda or a factory method. +During runtime the <cite>:py:class:Accumulator</cite> will be constructed for every +instance in which this UDAF is used. The following examples are all valid. +<a href="#id5"><span class="problematic" id="id6">``</span></a>` +import pyarrow as pa +import pyarrow.compute as pc</p> +<dl class="simple"> +<dt>class Summarize(Accumulator):</dt><dd><dl class="simple"> +<dt>def __init__(self, bias: float = 0.0):</dt><dd><p>self._sum = pa.scalar(bias)</p> +</dd> +<dt>def state(self) -> list[pa.Scalar]:</dt><dd><p>return [self._sum]</p> +</dd> +<dt>def update(self, values: pa.Array) -> None:</dt><dd><p>self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py())</p> +</dd> +<dt>def merge(self, states: list[pa.Array]) -> None:</dt><dd><p>self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py())</p> +</dd> +<dt>def evaluate(self) -> pa.Scalar:</dt><dd><p>return self._sum</p> +</dd> +</dl> +</dd> +<dt>def sum_bias_10() -> Summarize:</dt><dd><p>return Summarize(10.0)</p> +</dd> +<dt>udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()],</dt><dd><p>“immutable”)</p> +</dd> +<dt>udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()],</dt><dd><p>“immutable”)</p> +</dd> +<dt>udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),</dt><dd><p>[pa.float64()], “immutable”)</p> +</dd> +</dl> +<p><a href="#id7"><span class="problematic" id="id8">``</span></a><a href="#id9"><span class="problematic" id="id10">`</span></a></p> +</div></blockquote> +<dl> +<dt><strong>Decorator example:</strong></dt><dd><p><a href="#id11"><span class="problematic" id="id12">``</span></a>` +@udaf(pa.float64(), pa.float64(), [pa.float64()], “immutable”) +def udf4() -> Summarize:</p> +<blockquote> +<div><p>return Summarize(10.0)</p> +</div></blockquote> +<p><a href="#id13"><span class="problematic" id="id14">``</span></a><a href="#id15"><span class="problematic" id="id16">`</span></a></p> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>accum</strong> – The accumulator python function.</p></li> +<li><p><strong>accum</strong> – The accumulator python function. <strong>Only needed when calling as a +function. Skip this argument when using `udaf` as a decorator.</strong></p></li> <li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">accum</span></code>.</p></li> <li><p><strong>return_type</strong> – The data type of the return value.</p></li> <li><p><strong>state_type</strong> – The data types of the intermediate accumulation.</p></li> @@ -2104,7 +2274,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.ExecutionPlan.children"> -<span class="sig-name descname"><span class="pre">children</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.ExecutionPlan" title="datafusion.ExecutionPlan"><span class="pre">ExecutionPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.ExecutionPlan.children" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">children</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.ExecutionPlan" title="datafusion.ExecutionPlan"><span class="pre">ExecutionPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.ExecutionPlan.children" title="Link to this definition">¶</a></dt> <dd><p>Get a list of children <cite>ExecutionPlan</cite> that act as inputs to this plan.</p> <p>The returned list will be empty for leaf nodes such as scans, will contain a single value for unary nodes, or two values for binary nodes (such as joins).</p> @@ -2151,7 +2321,7 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.Expr"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">Expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.expr.Expr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.Expr" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">Expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.expr.RawExpr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.Expr" title="Link to this definition">¶</a></dt> <dd><p>Expression object.</p> <p>Expressions are one of the core concepts in DataFusion. See <a class="reference internal" href="../../user-guide/common-operations/expressions.html#expressions"><span class="std std-ref">Expressions</span></a> in the online documentation for more information.</p> @@ -2171,7 +2341,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.Expr.__eq__"> -<span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.__eq__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.__eq__" title="Link to this definition">¶</a></dt> <dd><p>Equal to.</p> <p>Accepts either an expression or any valid PyArrow scalar literal value.</p> </dd></dl> @@ -2235,7 +2405,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.Expr.__ne__"> -<span class="sig-name descname"><span class="pre">__ne__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.__ne__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__ne__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rhs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.__ne__" title="Link to this definition">¶</a></dt> <dd><p>Not equal to.</p> <p>Accepts either an expression or any valid PyArrow scalar literal value.</p> </dd></dl> @@ -2301,7 +2471,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.Expr.cast"> -<span class="sig-name descname"><span class="pre">cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">to</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">float</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Type</span><span class="p"><span class="pre">[</span></span><span class="pre">bool</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.cast" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">to</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">type</span><span class="p"><span class="pre">[</span></span><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">bool</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.Expr" title="datafusion.Expr"><span class="pre">Expr</span></a></span></span><a class="headerlink" href="#datafusion.Expr.cast" title="Link to this definition">¶</a></dt> <dd><p>Cast to a new data type.</p> </dd></dl> @@ -2554,7 +2724,7 @@ <dl class="py attribute"> <dt class="sig sig-object py" id="datafusion.Expr._to_pyarrow_types"> -<span class="sig-name descname"><span class="pre">_to_pyarrow_types</span></span><a class="headerlink" href="#datafusion.Expr._to_pyarrow_types" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">_to_pyarrow_types</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">ClassVar</span><span class="p"><span class="pre">[</span></span><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">type</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#datafusion.Expr._to_pyarrow_types" title="Link to this definition">¶</a></dt> <dd></dd></dl> <dl class="py attribute"> @@ -2618,7 +2788,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.LogicalPlan.inputs"> -<span class="sig-name descname"><span class="pre">inputs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.LogicalPlan" title="datafusion.LogicalPlan"><span class="pre">LogicalPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.LogicalPlan.inputs" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">inputs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.LogicalPlan" title="datafusion.LogicalPlan"><span class="pre">LogicalPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.LogicalPlan.inputs" title="Link to this definition">¶</a></dt> <dd><p>Returns the list of inputs to the logical plan.</p> </dd></dl> @@ -2645,13 +2815,13 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.RecordBatch"> <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">RecordBatch</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">record_batch</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.RecordBatch</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.RecordBatch" title="Link to this definition">¶</a></dt> -<dd><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p> +<dd><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p> <p>This constructor is generally not called by the end user.</p> <p>See the <a class="reference internal" href="#datafusion.RecordBatchStream" title="datafusion.RecordBatchStream"><code class="xref py py-class docutils literal notranslate"><span class="pre">RecordBatchStream</span></code></a> iterator for generating this class.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.RecordBatch.to_pyarrow"> <span class="sig-name descname"><span class="pre">to_pyarrow</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.RecordBatch</span></span></span><a class="headerlink" href="#datafusion.RecordBatch.to_pyarrow" title="Link to this definition">¶</a></dt> -<dd><p>Convert to <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p> +<dd><p>Convert to <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p> </dd></dl> <dl class="py attribute"> @@ -2912,7 +3082,7 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.ScalarUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">ScalarUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.ScalarUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">ScalarUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.ScalarUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing scalar user-defined functions (UDF).</p> <p>Scalar UDFs operate on a row by row basis. See also <a class="reference internal" href="#datafusion.AggregateUDF" title="datafusion.AggregateUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">AggregateUDF</span></code></a> for operating on a group of rows.</p> @@ -2928,28 +3098,56 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.ScalarUDF.udf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.ScalarUDF" title="datafusion.ScalarUDF"><span class="pre">ScalarUDF</span></a></span></span><a class="headerlink" href="#datafusion.ScalarUDF.udf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Function.</p> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.ScalarUDF" title="datafusion.ScalarUDF"><span class="pre">ScalarUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.ScalarUDF.udf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.ScalarUDF" title="datafusion.ScalarUDF"><span class="pre">ScalarUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Function (UDF).</p> +<p>This class can be used both as a <strong>function</strong> and as a <strong>decorator</strong>.</p> +<dl class="simple"> +<dt>Usage:</dt><dd><ul class="simple"> +<li><p><strong>As a function</strong>: Call <cite>udf(func, input_types, return_type, volatility, +name)</cite>.</p></li> +<li><p><strong>As a decorator</strong>: Use <cite>@udf(input_types, return_type, volatility, +name)</cite>. In this case, do <strong>not</strong> pass <cite>func</cite> explicitly.</p></li> +</ul> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>func</strong> – A callable python function.</p></li> -<li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">func</span></code>. This list -must be of the same length as the number of arguments.</p></li> -<li><p><strong>return_type</strong> – The data type of the return value from the python -function.</p></li> -<li><p><strong>volatility</strong> – See <code class="docutils literal notranslate"><span class="pre">Volatility</span></code> for allowed values.</p></li> -<li><p><strong>name</strong> – A descriptive name for the function.</p></li> +<li><p><strong>func</strong> (<em>Callable</em><em>, </em><em>optional</em>) – <strong>Only needed when calling as a function.</strong> +Skip this argument when using <cite>udf</cite> as a decorator.</p></li> +<li><p><strong>input_types</strong> (<em>list</em><em>[</em><em>pa.DataType</em><em>]</em>) – The data types of the arguments +to <cite>func</cite>. This list must be of the same length as the number of +arguments.</p></li> +<li><p><strong>return_type</strong> (<em>_R</em>) – The data type of the return value from the function.</p></li> +<li><p><strong>volatility</strong> (<a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><em>Volatility</em></a><em> | </em><em>str</em>) – See <cite>Volatility</cite> for allowed values.</p></li> +<li><p><strong>name</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em>) – A descriptive name for the function.</p></li> </ul> </dd> <dt class="field-even">Returns<span class="colon">:</span></dt> -<dd class="field-even"><p><dl class="simple"> -<dt>A user-defined aggregate function, which can be used in either data</dt><dd><p>aggregation or window function calls.</p> +<dd class="field-even"><p>A user-defined function that can be used in SQL expressions, +data aggregation, or window function calls.</p> </dd> </dl> -</p> -</dd> -</dl> +<p class="rubric">Example</p> +<p><strong>Using `udf` as a function:</strong> +<a href="#id17"><span class="problematic" id="id18">``</span></a>` +def double_func(x):</p> +<blockquote> +<div><p>return x * 2</p> +</div></blockquote> +<p>double_udf = udf(double_func, [pa.int32()], pa.int32(), +“volatile”, “double_it”) +<a href="#id19"><span class="problematic" id="id20">``</span></a><a href="#id21"><span class="problematic" id="id22">`</span></a></p> +<p><strong>Using `udf` as a decorator:</strong> +<a href="#id23"><span class="problematic" id="id24">``</span></a>` +@udf([pa.int32()], pa.int32(), “volatile”, “double_it”) +def double_udf(x):</p> +<blockquote> +<div><p>return x * 2</p> +</div></blockquote> +<p><a href="#id25"><span class="problematic" id="id26">``</span></a><a href="#id27"><span class="problematic" id="id28">`</span></a></p> </dd></dl> <dl class="py attribute"> @@ -3172,18 +3370,18 @@ <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">Table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.Table</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.Table" title="Link to this definition">¶</a></dt> <dd><p>DataFusion table.</p> <p>This constructor is not typically called by the end user.</p> -<dl class="py method"> -<dt class="sig sig-object py" id="datafusion.Table.schema"> -<span class="sig-name descname"><span class="pre">schema</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Schema</span></span></span><a class="headerlink" href="#datafusion.Table.schema" title="Link to this definition">¶</a></dt> -<dd><p>Returns the schema associated with this table.</p> -</dd></dl> - <dl class="py property"> <dt class="sig sig-object py" id="datafusion.Table.kind"> <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">kind</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#datafusion.Table.kind" title="Link to this definition">¶</a></dt> <dd><p>Returns the kind of table.</p> </dd></dl> +<dl class="py property"> +<dt class="sig sig-object py" id="datafusion.Table.schema"> +<em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">schema</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">pyarrow.Schema</span></em><a class="headerlink" href="#datafusion.Table.schema" title="Link to this definition">¶</a></dt> +<dd><p>Returns the schema associated with this table.</p> +</dd></dl> + <dl class="py attribute"> <dt class="sig sig-object py" id="datafusion.Table.table"> <span class="sig-name descname"><span class="pre">table</span></span><a class="headerlink" href="#datafusion.Table.table" title="Link to this definition">¶</a></dt> @@ -3223,7 +3421,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.WindowFrame.get_upper_bound"> -<span class="sig-name descname"><span class="pre">get_upper_bound</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.WindowFrame.get_upper_bound" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">get_upper_bound</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="expr/index.html#datafusion.expr.WindowFrameBound" title="datafusion.expr.WindowFrameBound"><span class="pre">WindowFrameBound</span></a></span></span><a class="headerlink" href="#datafusion.WindowFrame.get_upper_bound" title="Link to this definition">¶</a></dt> <dd><p>Returns end bound.</p> </dd></dl> @@ -3236,12 +3434,12 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.WindowUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">WindowUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.WindowUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">WindowUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.WindowUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing window user-defined functions (UDF).</p> <p>Window UDFs operate on a partition of rows. See also <a class="reference internal" href="#datafusion.ScalarUDF" title="datafusion.ScalarUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">ScalarUDF</span></code></a> for operating on a row by row basis.</p> <p>Instantiate a user-defined window function (UDWF).</p> -<p>See <a class="reference internal" href="#datafusion.WindowUDF.udwf" title="datafusion.WindowUDF.udwf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udwf()</span></code></a> for a convenience function and argument +<p>See <a class="reference internal" href="#datafusion.udwf" title="datafusion.udwf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udwf()</span></code></a> for a convenience function and argument descriptions.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.WindowUDF.__call__"> @@ -3252,44 +3450,86 @@ </dd></dl> <dl class="py method"> +<dt class="sig sig-object py" id="datafusion.WindowUDF._create_window_udf"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_create_window_udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.WindowUDF" title="datafusion.WindowUDF"><span class="pre">WindowUDF</span></a></span></span><a class="headerlink" href="#datafusion.WindowUDF._create_window_udf" title="Link to this definition">¶</a></dt> +<dd><p>Create a WindowUDF instance from function arguments.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.WindowUDF._create_window_udf_decorator"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_create_window_udf_decorator</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.WindowUDF._create_window_udf_decorator" title="Link to this definition">¶</a></dt> +<dd><p>Create a decorator for a WindowUDF.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.WindowUDF._get_default_name"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_get_default_name</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.WindowUDF._get_default_name" title="Link to this definition">¶</a></dt> +<dd><p>Get the default name for a function based on its attributes.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.WindowUDF._normalize_input_types"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_normalize_input_types</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.WindowUDF._normalize_input_types" title="Link to this definition">¶</a></dt> +<dd><p>Convert a single DataType to a list if needed.</p> +</dd></dl> + +<dl class="py method"> <dt class="sig sig-object py" id="datafusion.WindowUDF.udwf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.WindowUDF" title="datafusion.WindowUDF"><span class="pre">WindowUDF</span></a></span></span><a class="headerlink" href="#datafusion.WindowUDF.udwf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Window Function.</p> -<p>If your <code class="xref py py-class docutils literal notranslate"><span class="pre">WindowEvaluator</span></code> can be instantiated with no arguments, you -can simply pass it’s type as <code class="docutils literal notranslate"><span class="pre">func</span></code>. If you need to pass additional arguments -to it’s constructor, you can define a lambda or a factory method. During runtime -the <code class="xref py py-class docutils literal notranslate"><span class="pre">WindowEvaluator</span></code> will be constructed for every instance in -which this UDWF is used. The following examples are all valid.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> - -<span class="k">class</span><span class="w"> </span><span class="nc">BiasedNumbers</span><span class="p">(</span><span class="n">WindowEvaluator</span><span class="p">):</span> - <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">start</span> <span class="o">=</span> <span class="n">start</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">evaluate_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">values</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">],</span> <span class="n">num_rows</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span> - <span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span> <span class="o">+</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_rows</span><span class="p">)])</span> - -<span class="k">def</span><span class="w"> </span><span class="nf">bias_10</span><span class="p">()</span> <span class="o">-></span> <span class="n">BiasedNumbers</span><span class="p">:</span> - <span class="k">return</span> <span class="n">BiasedNumbers</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> - -<span class="n">udwf1</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="n">BiasedNumbers</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udwf2</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="n">bias_10</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udwf3</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="k">lambda</span><span class="p">:</span> <span class="n">BiasedNumbers</span><span class="p">(</span><span class="mi">20</span><span class="p">),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -</pre></div> -</div> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.WindowUDF" title="datafusion.WindowUDF"><span class="pre">WindowUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.WindowUDF.udwf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="udf/index.html#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="udf/index.html#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.WindowUDF" title="datafusion.WindowUDF"><span class="pre">WindowUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Window Function (UDWF).</p> +<p>This class can be used both as a <strong>function</strong> and as a <strong>decorator</strong>.</p> +<dl> +<dt>Usage:</dt><dd><ul class="simple"> +<li><p><strong>As a function</strong>: Call <cite>udwf(func, input_types, return_type, volatility, +name)</cite>.</p></li> +<li><p><strong>As a decorator</strong>: Use <cite>@udwf(input_types, return_type, volatility, +name)</cite>. When using <cite>udwf</cite> as a decorator, <strong>do not pass `func` +explicitly</strong>.</p></li> +</ul> +</dd> +<dt><strong>Function example:</strong></dt><dd><p><a href="#id29"><span class="problematic" id="id30">``</span></a>` +import pyarrow as pa</p> +<dl class="simple"> +<dt>class BiasedNumbers(WindowEvaluator):</dt><dd><dl class="simple"> +<dt>def __init__(self, start: int = 0) -> None:</dt><dd><p>self.start = start</p> +</dd> +<dt>def evaluate_all(self, values: list[pa.Array],</dt><dd><p>num_rows: int) -> pa.Array: +return pa.array([self.start + i for i in range(num_rows)])</p> +</dd> +</dl> +</dd> +<dt>def bias_10() -> BiasedNumbers:</dt><dd><p>return BiasedNumbers(10)</p> +</dd> +</dl> +<p>udwf1 = udwf(BiasedNumbers, pa.int64(), pa.int64(), “immutable”) +udwf2 = udwf(bias_10, pa.int64(), pa.int64(), “immutable”) +udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), “immutable”)</p> +<p><a href="#id31"><span class="problematic" id="id32">``</span></a><a href="#id33"><span class="problematic" id="id34">`</span></a></p> +</dd> +<dt><strong>Decorator example:</strong></dt><dd><p><a href="#id35"><span class="problematic" id="id36">``</span></a>` +@udwf(pa.int64(), pa.int64(), “immutable”) +def biased_numbers() -> BiasedNumbers:</p> +<blockquote> +<div><p>return BiasedNumbers(10)</p> +</div></blockquote> +<p><a href="#id37"><span class="problematic" id="id38">``</span></a><a href="#id39"><span class="problematic" id="id40">`</span></a></p> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>func</strong> – A callable to create the window function.</p></li> -<li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li> +<li><p><strong>func</strong> – <strong>Only needed when calling as a function. Skip this argument when +using `udwf` as a decorator.</strong></p></li> +<li><p><strong>input_types</strong> – The data types of the arguments.</p></li> <li><p><strong>return_type</strong> – The data type of the return value.</p></li> <li><p><strong>volatility</strong> – See <code class="xref py py-class docutils literal notranslate"><span class="pre">Volatility</span></code> for allowed values.</p></li> -<li><p><strong>arguments</strong> – A list of arguments to pass in to the __init__ method for accum.</p></li> <li><p><strong>name</strong> – A descriptive name for the function.</p></li> </ul> </dd> <dt class="field-even">Returns<span class="colon">:</span></dt> -<dd class="field-even"><p>A user-defined window function.</p> +<dd class="field-even"><p>A user-defined window function that can be used in window function calls.</p> </dd> </dl> </dd></dl> @@ -3303,33 +3543,158 @@ <dl class="py function"> <dt class="sig sig-object py" id="datafusion.col"> -<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">col</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.col" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">col</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.col" title="Link to this definition">¶</a></dt> <dd><p>Create a column expression.</p> </dd></dl> <dl class="py function"> <dt class="sig sig-object py" id="datafusion.column"> -<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.column" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.column" title="Link to this definition">¶</a></dt> <dd><p>Create a column expression.</p> </dd></dl> <dl class="py function"> <dt class="sig sig-object py" id="datafusion.lit"> -<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">lit</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.lit" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">lit</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.lit" title="Link to this definition">¶</a></dt> <dd><p>Create a literal expression.</p> </dd></dl> <dl class="py function"> <dt class="sig sig-object py" id="datafusion.literal"> -<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">literal</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.literal" title="Link to this definition">¶</a></dt> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">literal</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.literal" title="Link to this definition">¶</a></dt> <dd><p>Create a literal expression.</p> </dd></dl> +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.read_avro"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.avro'</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.read_avro" title="Link to this definition">¶</a></dt> +<dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the Avro file.</p></li> +<li><p><strong>schema</strong> – The data source schema.</p></li> +<li><p><strong>file_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_extension</strong> – File extension to select.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read Avro file</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.read_csv"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">read_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pathlib.Path</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">has_header</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">delimiter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">','</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema_infer_max_records</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.csv'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_compression_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.read_csv" title="Link to this definition">¶</a></dt> +<dd><p>Read a CSV data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the CSV file</p></li> +<li><p><strong>schema</strong> – An optional schema representing the CSV files. If None, the +CSV reader will try to infer it based on data in file.</p></li> +<li><p><strong>has_header</strong> – Whether the CSV file have a header. If schema inference +is run on a file with no headers, default column names are +created.</p></li> +<li><p><strong>delimiter</strong> – An optional column delimiter.</p></li> +<li><p><strong>schema_infer_max_records</strong> – Maximum number of rows to read from CSV +files for schema inference if needed.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_compression_type</strong> – File compression type.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read CSV files</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.read_json"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">read_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema_infer_max_records</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_compression_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.read_json" title="Link to this definition">¶</a></dt> +<dd><p>Read a line-delimited JSON data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the JSON file.</p></li> +<li><p><strong>schema</strong> – The data source schema.</p></li> +<li><p><strong>schema_infer_max_records</strong> – Maximum number of rows to read from JSON +files for schema inference if needed.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_compression_type</strong> – File compression type.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read JSON files.</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.read_parquet"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.read_parquet" title="Link to this definition">¶</a></dt> +<dd><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the Parquet file.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>parquet_pruning</strong> – Whether the parquet reader should use the predicate +to prune row groups.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>skip_metadata</strong> – Whether the parquet reader should skip any metadata +that may be in the file schema. This can help avoid schema +conflicts due to metadata.</p></li> +<li><p><strong>schema</strong> – An optional schema representing the parquet files. If None, +the parquet reader will try to infer it based on data in the +file.</p></li> +<li><p><strong>file_sort_order</strong> – Sort order for the file.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read Parquet files</p> +</dd> +</dl> +</dd></dl> + <dl class="py data"> <dt class="sig sig-object py" id="datafusion.DFSchema"> <span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">DFSchema</span></span><a class="headerlink" href="#datafusion.DFSchema" title="Link to this definition">¶</a></dt> <dd></dd></dl> +<dl class="py data"> +<dt class="sig sig-object py" id="datafusion.udaf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">udaf</span></span><a class="headerlink" href="#datafusion.udaf" title="Link to this definition">¶</a></dt> +<dd></dd></dl> + +<dl class="py data"> +<dt class="sig sig-object py" id="datafusion.udf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">udf</span></span><a class="headerlink" href="#datafusion.udf" title="Link to this definition">¶</a></dt> +<dd></dd></dl> + +<dl class="py data"> +<dt class="sig sig-object py" id="datafusion.udwf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.</span></span><span class="sig-name descname"><span class="pre">udwf</span></span><a class="headerlink" href="#datafusion.udwf" title="Link to this definition">¶</a></dt> +<dd></dd></dl> + </section> </section>
diff --git a/autoapi/datafusion/input/base/index.html b/autoapi/datafusion/input/base/index.html index cf44b1a..5c55d0b 100644 --- a/autoapi/datafusion/input/base/index.html +++ b/autoapi/datafusion/input/base/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../object_store/index.html"> datafusion.object_store </a> @@ -450,13 +465,13 @@ to obtain the SqlTable information from the custom datasource.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.input.base.BaseInputSource.build_table"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">build_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwarg</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">datafusion.common.SqlTable</span></span></span><a class="headerlink" href="#datafusion.input.base.BaseInputSource.build_table" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">build_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwarg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">datafusion.common.SqlTable</span></span></span><a class="headerlink" href="#datafusion.input.base.BaseInputSource.build_table" title="Link to this definition">¶</a></dt> <dd><p>Create a table from the input source.</p> </dd></dl> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.input.base.BaseInputSource.is_correct_input"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">is_correct_input</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.input.base.BaseInputSource.is_correct_input" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">is_correct_input</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.input.base.BaseInputSource.is_correct_input" title="Link to this definition">¶</a></dt> <dd><p>Returns <cite>True</cite> if the input is valid.</p> </dd></dl>
diff --git a/autoapi/datafusion/input/index.html b/autoapi/datafusion/input/index.html index 896164d..1f36814 100644 --- a/autoapi/datafusion/input/index.html +++ b/autoapi/datafusion/input/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -365,6 +380,47 @@ Submodules </a> </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#classes"> + Classes + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#package-contents"> + Package Contents + </a> + <ul class="visible nav section-nav flex-column"> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.input.LocationInputPlugin"> + <code class="docutils literal notranslate"> + <span class="pre"> + LocationInputPlugin + </span> + </code> + </a> + <ul class="nav section-nav flex-column"> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.input.LocationInputPlugin.build_table"> + <code class="docutils literal notranslate"> + <span class="pre"> + LocationInputPlugin.build_table() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.input.LocationInputPlugin.is_correct_input"> + <code class="docutils literal notranslate"> + <span class="pre"> + LocationInputPlugin.is_correct_input() + </span> + </code> + </a> + </li> + </ul> + </li> + </ul> + </li> </ul> </nav> @@ -399,6 +455,39 @@ </ul> </div> </section> +<section id="classes"> +<h2>Classes<a class="headerlink" href="#classes" title="Link to this heading">¶</a></h2> +<table class="autosummary longtable table autosummary"> +<tbody> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.input.LocationInputPlugin" title="datafusion.input.LocationInputPlugin"><code class="xref py py-obj docutils literal notranslate"><span class="pre">LocationInputPlugin</span></code></a></p></td> +<td><p>Input Plugin for everything.</p></td> +</tr> +</tbody> +</table> +</section> +<section id="package-contents"> +<h2>Package Contents<a class="headerlink" href="#package-contents" title="Link to this heading">¶</a></h2> +<dl class="py class"> +<dt class="sig sig-object py" id="datafusion.input.LocationInputPlugin"> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.input.</span></span><span class="sig-name descname"><span class="pre">LocationInputPlugin</span></span><a class="headerlink" href="#datafusion.input.LocationInputPlugin" title="Link to this definition">¶</a></dt> +<dd><p>Bases: <a class="reference internal" href="base/index.html#datafusion.input.base.BaseInputSource" title="datafusion.input.base.BaseInputSource"><code class="xref py py-obj docutils literal notranslate"><span class="pre">datafusion.input.base.BaseInputSource</span></code></a></p> +<p>Input Plugin for everything.</p> +<p>This can be read in from a file (on disk, remote etc.).</p> +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.input.LocationInputPlugin.build_table"> +<span class="sig-name descname"><span class="pre">build_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">datafusion.common.SqlTable</span></span></span><a class="headerlink" href="#datafusion.input.LocationInputPlugin.build_table" title="Link to this definition">¶</a></dt> +<dd><p>Create a table from the input source.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.input.LocationInputPlugin.is_correct_input"> +<span class="sig-name descname"><span class="pre">is_correct_input</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.input.LocationInputPlugin.is_correct_input" title="Link to this definition">¶</a></dt> +<dd><p>Returns <cite>True</cite> if the input is valid.</p> +</dd></dl> + +</dd></dl> + +</section> </section>
diff --git a/autoapi/datafusion/input/location/index.html b/autoapi/datafusion/input/location/index.html index 9d46682..2be4274 100644 --- a/autoapi/datafusion/input/location/index.html +++ b/autoapi/datafusion/input/location/index.html
@@ -34,7 +34,7 @@ <script src="../../../../_static/sphinx_highlight.js?v=dc90522c"></script> <link rel="index" title="Index" href="../../../../genindex.html" /> <link rel="search" title="Search" href="../../../../search.html" /> - <link rel="next" title="datafusion.object_store" href="../../object_store/index.html" /> + <link rel="next" title="datafusion.io" href="../../io/index.html" /> <link rel="prev" title="datafusion.input.base" href="../base/index.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../object_store/index.html"> datafusion.object_store </a> @@ -445,13 +460,13 @@ <p>This can be read in from a file (on disk, remote etc.).</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.input.location.LocationInputPlugin.build_table"> -<span class="sig-name descname"><span class="pre">build_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_file</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">datafusion.common.SqlTable</span></span></span><a class="headerlink" href="#datafusion.input.location.LocationInputPlugin.build_table" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">build_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">datafusion.common.SqlTable</span></span></span><a class="headerlink" href="#datafusion.input.location.LocationInputPlugin.build_table" title="Link to this definition">¶</a></dt> <dd><p>Create a table from the input source.</p> </dd></dl> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.input.location.LocationInputPlugin.is_correct_input"> -<span class="sig-name descname"><span class="pre">is_correct_input</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.input.location.LocationInputPlugin.is_correct_input" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">is_correct_input</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_item</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.input.location.LocationInputPlugin.is_correct_input" title="Link to this definition">¶</a></dt> <dd><p>Returns <cite>True</cite> if the input is valid.</p> </dd></dl> @@ -473,10 +488,10 @@ <p class="prev-next-title">datafusion.input.base</p> </div> </a> - <a class='right-next' id="next-link" href="../../object_store/index.html" title="next page"> + <a class='right-next' id="next-link" href="../../io/index.html" title="next page"> <div class="prev-next-info"> <p class="prev-next-subtitle">next</p> - <p class="prev-next-title">datafusion.object_store</p> + <p class="prev-next-title">datafusion.io</p> </div> <i class="fas fa-angle-right"></i> </a>
diff --git a/autoapi/datafusion/io/index.html b/autoapi/datafusion/io/index.html new file mode 100644 index 0000000..3ce9435 --- /dev/null +++ b/autoapi/datafusion/io/index.html
@@ -0,0 +1,639 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> + + <title>datafusion.io — Apache Arrow DataFusion documentation</title> + + <link href="../../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> +<link href="../../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> + + + <link rel="stylesheet" + href="../../../_static/vendor/fontawesome/5.13.0/css/all.min.css"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../../../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../../../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2"> + + + + + + <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" /> + <link rel="stylesheet" type="text/css" href="../../../_static/styles/pydata-sphinx-theme.css?v=1140d252" /> + <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css?v=4ae1632d" /> + <link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css?v=dca7052a" /> + + <link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"> + + <script src="../../../_static/documentation_options.js?v=8a448e45"></script> + <script src="../../../_static/doctools.js?v=9bcbadda"></script> + <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> + <link rel="index" title="Index" href="../../../genindex.html" /> + <link rel="search" title="Search" href="../../../search.html" /> + <link rel="next" title="datafusion.object_store" href="../object_store/index.html" /> + <link rel="prev" title="datafusion.input.location" href="../input/location/index.html" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <meta name="docsearch:language" content="en"> + + + <!-- Google Analytics --> + + </head> + <body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80"> + + <div class="container-fluid" id="banner"></div> + + + + + <div class="container-xl"> + <div class="row"> + + + <!-- Only show if we have sidebars configured, else just a small margin --> + <div class="col-12 col-md-3 bd-sidebar"> + <div class="sidebar-start-items"> +<a class="navbar-brand" href="../../../index.html"> + <img src="../../../_static/images/2x_bgwhite_original.png" class="logo" alt="logo"> +</a> + +<form class="bd-search d-flex align-items-center" action="../../../search.html" method="get"> + <i class="icon fas fa-search"></i> + <input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" > +</form> + +<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation"> + <div class="bd-toc-item active"> + + <p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + LINKS + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python"> + Github and Issue Tracker + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/"> + Rust's API Docs + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md"> + Code of conduct + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples"> + Examples + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + USER GUIDE + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="../../../user-guide/introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../user-guide/basics.html"> + Concepts + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../user-guide/data-sources.html"> + Data Sources + </a> + </li> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../../../user-guide/common-operations/index.html"> + Common Operations + </a> + <input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/> + <label for="toctree-checkbox-1"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> + Basic Operations + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/select-and-filter.html"> + Column Selections + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/expressions.html"> + Expressions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/joins.html"> + Joins + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/functions.html"> + Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/aggregations.html"> + Aggregation + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/windows.html"> + Window Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html"> + User-Defined Functions + </a> + </li> + </ul> + </li> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../../../user-guide/io/index.html"> + IO + </a> + <input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/> + <label for="toctree-checkbox-2"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/arrow.html"> + Arrow + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/avro.html"> + Avro + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/csv.html"> + CSV + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/json.html"> + JSON + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/parquet.html"> + Parquet + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/io/table_provider.html"> + Custom Table Provider + </a> + </li> + </ul> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../user-guide/configuration.html"> + Configuration + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../user-guide/sql.html"> + SQL + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + CONTRIBUTOR GUIDE + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + API + </span> +</p> +<ul class="current nav bd-sidenav"> + <li class="toctree-l1 current active has-children"> + <a class="reference internal" href="../../index.html"> + API Reference + </a> + <input checked="" class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/> + <label for="toctree-checkbox-3"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul class="current"> + <li class="toctree-l2 current active has-children"> + <a class="reference internal" href="../index.html"> + datafusion + </a> + <input checked="" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/> + <label for="toctree-checkbox-4"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul class="current"> + <li class="toctree-l3"> + <a class="reference internal" href="../catalog/index.html"> + datafusion.catalog + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../context/index.html"> + datafusion.context + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../dataframe/index.html"> + datafusion.dataframe + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../expr/index.html"> + datafusion.expr + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../functions/index.html"> + datafusion.functions + </a> + </li> + <li class="toctree-l3 has-children"> + <a class="reference internal" href="../input/index.html"> + datafusion.input + </a> + <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/> + <label for="toctree-checkbox-5"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l4"> + <a class="reference internal" href="../input/base/index.html"> + datafusion.input.base + </a> + </li> + <li class="toctree-l4"> + <a class="reference internal" href="../input/location/index.html"> + datafusion.input.location + </a> + </li> + </ul> + </li> + <li class="toctree-l3 current active"> + <a class="current reference internal" href="#"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../object_store/index.html"> + datafusion.object_store + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../plan/index.html"> + datafusion.plan + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../record_batch/index.html"> + datafusion.record_batch + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../substrait/index.html"> + datafusion.substrait + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../udf/index.html"> + datafusion.udf + </a> + </li> + </ul> + </li> + </ul> + </li> +</ul> + + + </div> +</nav> + </div> + <div class="sidebar-end-items"> + </div> + </div> + + + + + <div class="d-none d-xl-block col-xl-2 bd-toc"> + + + <div class="toc-item"> + +<div class="tocsection onthispage pt-5 pb-3"> + <i class="fas fa-list"></i> On this page +</div> + +<nav id="bd-toc-nav"> + <ul class="visible nav section-nav flex-column"> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#functions"> + Functions + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#module-contents"> + Module Contents + </a> + <ul class="visible nav section-nav flex-column"> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.io.read_avro"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_avro() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.io.read_csv"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_csv() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.io.read_json"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_json() + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.io.read_parquet"> + <code class="docutils literal notranslate"> + <span class="pre"> + read_parquet() + </span> + </code> + </a> + </li> + </ul> + </li> +</ul> + +</nav> + </div> + + <div class="toc-item"> + + </div> + + + </div> + + + + + + + <main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main"> + + <div> + + <section id="module-datafusion.io"> +<span id="datafusion-io"></span><h1>datafusion.io<a class="headerlink" href="#module-datafusion.io" title="Link to this heading">¶</a></h1> +<p>IO read functions using global context.</p> +<section id="functions"> +<h2>Functions<a class="headerlink" href="#functions" title="Link to this heading">¶</a></h2> +<table class="autosummary longtable table autosummary"> +<tbody> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.io.read_avro" title="datafusion.io.read_avro"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_avro</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.io.read_csv" title="datafusion.io.read_csv"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_csv</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a CSV data source.</p></td> +</tr> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.io.read_json" title="datafusion.io.read_json"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_json</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a line-delimited JSON data source.</p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.io.read_parquet" title="datafusion.io.read_parquet"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_parquet</span></code></a>(→ datafusion.dataframe.DataFrame)</p></td> +<td><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p></td> +</tr> +</tbody> +</table> +</section> +<section id="module-contents"> +<h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">¶</a></h2> +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.io.read_avro"> +<span class="sig-prename descclassname"><span class="pre">datafusion.io.</span></span><span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.avro'</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.io.read_avro" title="Link to this definition">¶</a></dt> +<dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the Avro file.</p></li> +<li><p><strong>schema</strong> – The data source schema.</p></li> +<li><p><strong>file_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_extension</strong> – File extension to select.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read Avro file</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.io.read_csv"> +<span class="sig-prename descclassname"><span class="pre">datafusion.io.</span></span><span class="sig-name descname"><span class="pre">read_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pathlib.Path</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">has_header</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">delimiter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">','</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema_infer_max_records</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.csv'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_compression_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.io.read_csv" title="Link to this definition">¶</a></dt> +<dd><p>Read a CSV data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the CSV file</p></li> +<li><p><strong>schema</strong> – An optional schema representing the CSV files. If None, the +CSV reader will try to infer it based on data in file.</p></li> +<li><p><strong>has_header</strong> – Whether the CSV file have a header. If schema inference +is run on a file with no headers, default column names are +created.</p></li> +<li><p><strong>delimiter</strong> – An optional column delimiter.</p></li> +<li><p><strong>schema_infer_max_records</strong> – Maximum number of rows to read from CSV +files for schema inference if needed.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_compression_type</strong> – File compression type.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read CSV files</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.io.read_json"> +<span class="sig-prename descclassname"><span class="pre">datafusion.io.</span></span><span class="sig-name descname"><span class="pre">read_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema_infer_max_records</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_compression_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.io.read_json" title="Link to this definition">¶</a></dt> +<dd><p>Read a line-delimited JSON data source.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the JSON file.</p></li> +<li><p><strong>schema</strong> – The data source schema.</p></li> +<li><p><strong>schema_infer_max_records</strong> – Maximum number of rows to read from JSON +files for schema inference if needed.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>file_compression_type</strong> – File compression type.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read JSON files.</p> +</dd> +</dl> +</dd></dl> + +<dl class="py function"> +<dt class="sig sig-object py" id="datafusion.io.read_parquet"> +<span class="sig-prename descclassname"><span class="pre">datafusion.io.</span></span><span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">parquet_pruning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'.parquet'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file_sort_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.io.read_parquet" title="Link to this definition">¶</a></dt> +<dd><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p> +<p>This function will use the global context. Any functions or tables registered +with another context may not be accessible when used with a DataFrame created +using this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters<span class="colon">:</span></dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>path</strong> – Path to the Parquet file.</p></li> +<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li> +<li><p><strong>parquet_pruning</strong> – Whether the parquet reader should use the predicate +to prune row groups.</p></li> +<li><p><strong>file_extension</strong> – File extension; only files with this extension are +selected for data input.</p></li> +<li><p><strong>skip_metadata</strong> – Whether the parquet reader should skip any metadata +that may be in the file schema. This can help avoid schema +conflicts due to metadata.</p></li> +<li><p><strong>schema</strong> – An optional schema representing the parquet files. If None, +the parquet reader will try to infer it based on data in the +file.</p></li> +<li><p><strong>file_sort_order</strong> – Sort order for the file.</p></li> +</ul> +</dd> +<dt class="field-even">Returns<span class="colon">:</span></dt> +<dd class="field-even"><p>DataFrame representation of the read Parquet files</p> +</dd> +</dl> +</dd></dl> + +</section> +</section> + + + </div> + + + <!-- Previous / next buttons --> +<div class='prev-next-area'> + <a class='left-prev' id="prev-link" href="../input/location/index.html" title="previous page"> + <i class="fas fa-angle-left"></i> + <div class="prev-next-info"> + <p class="prev-next-subtitle">previous</p> + <p class="prev-next-title">datafusion.input.location</p> + </div> + </a> + <a class='right-next' id="next-link" href="../object_store/index.html" title="next page"> + <div class="prev-next-info"> + <p class="prev-next-subtitle">next</p> + <p class="prev-next-title">datafusion.object_store</p> + </div> + <i class="fas fa-angle-right"></i> + </a> +</div> + + </main> + + + </div> + </div> + + <script src="../../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script> + +<!-- Based on pydata_sphinx_theme/footer.html --> +<footer class="footer mt-5 mt-md-0"> + <div class="container"> + + <div class="footer-item"> + <p class="copyright"> + © Copyright 2019-2024, Apache Software Foundation.<br> +</p> + </div> + + <div class="footer-item"> + <p class="sphinx-version"> +Created using <a href="http://sphinx-doc.org/">Sphinx</a> 8.1.3.<br> +</p> + </div> + + <div class="footer-item"> + <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> + <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> + </div> + </div> +</footer> + + + </body> +</html> \ No newline at end of file
diff --git a/autoapi/datafusion/object_store/index.html b/autoapi/datafusion/object_store/index.html index 71ad4a8..b27718f 100644 --- a/autoapi/datafusion/object_store/index.html +++ b/autoapi/datafusion/object_store/index.html
@@ -35,7 +35,7 @@ <link rel="index" title="Index" href="../../../genindex.html" /> <link rel="search" title="Search" href="../../../search.html" /> <link rel="next" title="datafusion.plan" href="../plan/index.html" /> - <link rel="prev" title="datafusion.input.location" href="../input/location/index.html" /> + <link rel="prev" title="datafusion.io" href="../io/index.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -307,6 +317,11 @@ </li> </ul> </li> + <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> <li class="toctree-l3 current active"> <a class="current reference internal" href="#"> datafusion.object_store @@ -499,11 +514,11 @@ <!-- Previous / next buttons --> <div class='prev-next-area'> - <a class='left-prev' id="prev-link" href="../input/location/index.html" title="previous page"> + <a class='left-prev' id="prev-link" href="../io/index.html" title="previous page"> <i class="fas fa-angle-left"></i> <div class="prev-next-info"> <p class="prev-next-subtitle">previous</p> - <p class="prev-next-title">datafusion.input.location</p> + <p class="prev-next-title">datafusion.io</p> </div> </a> <a class='right-next' id="next-link" href="../plan/index.html" title="next page">
diff --git a/autoapi/datafusion/plan/index.html b/autoapi/datafusion/plan/index.html index ddf72b3..df79d85 100644 --- a/autoapi/datafusion/plan/index.html +++ b/autoapi/datafusion/plan/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -608,7 +623,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.children"> -<span class="sig-name descname"><span class="pre">children</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">ExecutionPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.children" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">children</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">ExecutionPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.children" title="Link to this definition">¶</a></dt> <dd><p>Get a list of children <cite>ExecutionPlan</cite> that act as inputs to this plan.</p> <p>The returned list will be empty for leaf nodes such as scans, will contain a single value for unary nodes, or two values for binary nodes (such as joins).</p> @@ -707,7 +722,7 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.inputs"> -<span class="sig-name descname"><span class="pre">inputs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">LogicalPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.inputs" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">inputs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">LogicalPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.inputs" title="Link to this definition">¶</a></dt> <dd><p>Returns the list of inputs to the logical plan.</p> </dd></dl>
diff --git a/autoapi/datafusion/record_batch/index.html b/autoapi/datafusion/record_batch/index.html index f322281..edeb5af 100644 --- a/autoapi/datafusion/record_batch/index.html +++ b/autoapi/datafusion/record_batch/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -497,7 +512,7 @@ <table class="autosummary longtable table autosummary"> <tbody> <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.record_batch.RecordBatch" title="datafusion.record_batch.RecordBatch"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RecordBatch</span></code></a></p></td> -<td><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p></td> +<td><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p></td> </tr> <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.record_batch.RecordBatchStream" title="datafusion.record_batch.RecordBatchStream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RecordBatchStream</span></code></a></p></td> <td><p>This class represents a stream of record batches.</p></td> @@ -510,13 +525,13 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.record_batch.RecordBatch"> <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.record_batch.</span></span><span class="sig-name descname"><span class="pre">RecordBatch</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">record_batch</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.RecordBatch</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.record_batch.RecordBatch" title="Link to this definition">¶</a></dt> -<dd><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p> +<dd><p>This class is essentially a wrapper for <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p> <p>This constructor is generally not called by the end user.</p> <p>See the <a class="reference internal" href="#datafusion.record_batch.RecordBatchStream" title="datafusion.record_batch.RecordBatchStream"><code class="xref py py-class docutils literal notranslate"><span class="pre">RecordBatchStream</span></code></a> iterator for generating this class.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.record_batch.RecordBatch.to_pyarrow"> <span class="sig-name descname"><span class="pre">to_pyarrow</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.RecordBatch</span></span></span><a class="headerlink" href="#datafusion.record_batch.RecordBatch.to_pyarrow" title="Link to this definition">¶</a></dt> -<dd><p>Convert to <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>.</p> +<dd><p>Convert to <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code>.</p> </dd></dl> <dl class="py attribute">
diff --git a/autoapi/datafusion/substrait/index.html b/autoapi/datafusion/substrait/index.html index 9b71cde..87028da 100644 --- a/autoapi/datafusion/substrait/index.html +++ b/autoapi/datafusion/substrait/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a>
diff --git a/autoapi/datafusion/udf/index.html b/autoapi/datafusion/udf/index.html index 5452e59..9f0f674 100644 --- a/autoapi/datafusion/udf/index.html +++ b/autoapi/datafusion/udf/index.html
@@ -127,6 +127,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../../../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../../../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -232,6 +237,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -307,6 +317,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../object_store/index.html"> datafusion.object_store </a> @@ -655,6 +670,42 @@ </a> </li> <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.WindowUDF._create_window_udf"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._create_window_udf() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.WindowUDF._create_window_udf_decorator"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._create_window_udf_decorator() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.WindowUDF._get_default_name"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._get_default_name() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.WindowUDF._normalize_input_types"> + <code class="docutils literal notranslate"> + <span class="pre"> + WindowUDF._normalize_input_types() + </span> + </code> + </a> + </li> + <li class="toc-h4 nav-item toc-entry"> <a class="reference internal nav-link" href="#datafusion.udf.WindowUDF.udwf"> <code class="docutils literal notranslate"> <span class="pre"> @@ -684,10 +735,28 @@ </a> </li> <li class="toc-h3 nav-item toc-entry"> - <a class="reference internal nav-link" href="#datafusion.udf._W"> + <a class="reference internal nav-link" href="#datafusion.udf.udaf"> <code class="docutils literal notranslate"> <span class="pre"> - _W + udaf + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.udf"> + <code class="docutils literal notranslate"> + <span class="pre"> + udf + </span> + </code> + </a> + </li> + <li class="toc-h3 nav-item toc-entry"> + <a class="reference internal nav-link" href="#datafusion.udf.udwf"> + <code class="docutils literal notranslate"> + <span class="pre"> + udwf </span> </code> </a> @@ -725,7 +794,13 @@ <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.udf._R" title="datafusion.udf._R"><code class="xref py py-obj docutils literal notranslate"><span class="pre">_R</span></code></a></p></td> <td><p></p></td> </tr> -<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.udf._W" title="datafusion.udf._W"><code class="xref py py-obj docutils literal notranslate"><span class="pre">_W</span></code></a></p></td> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.udf.udaf" title="datafusion.udf.udaf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udaf</span></code></a></p></td> +<td><p></p></td> +</tr> +<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.udf.udf" title="datafusion.udf.udf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udf</span></code></a></p></td> +<td><p></p></td> +</tr> +<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.udf.udwf" title="datafusion.udf.udwf"><code class="xref py py-obj docutils literal notranslate"><span class="pre">udwf</span></code></a></p></td> <td><p></p></td> </tr> </tbody> @@ -770,13 +845,13 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.Accumulator.merge"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">merge</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">states</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Array</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.udf.Accumulator.merge" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">merge</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">states</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Array</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.udf.Accumulator.merge" title="Link to this definition">¶</a></dt> <dd><p>Merge a set of states.</p> </dd></dl> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.Accumulator.state"> -<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">state</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Scalar</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.Accumulator.state" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">state</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.Scalar</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.Accumulator.state" title="Link to this definition">¶</a></dt> <dd><p>Return the current state.</p> </dd></dl> @@ -790,12 +865,12 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.udf.AggregateUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">AggregateUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">accumulator</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.AggregateUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">AggregateUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">accumulator</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.AggregateUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing scalar user-defined functions (UDF).</p> <p>Aggregate UDFs operate on a group of rows and return a single value. See also <a class="reference internal" href="#datafusion.udf.ScalarUDF" title="datafusion.udf.ScalarUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">ScalarUDF</span></code></a> for operating on a row by row basis.</p> <p>Instantiate a user-defined aggregate function (UDAF).</p> -<p>See <a class="reference internal" href="#datafusion.udf.AggregateUDF.udaf" title="datafusion.udf.AggregateUDF.udaf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udaf()</span></code></a> for a convenience function and argument +<p>See <a class="reference internal" href="#datafusion.udf.udaf" title="datafusion.udf.udaf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udaf()</span></code></a> for a convenience function and argument descriptions.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.AggregateUDF.__call__"> @@ -807,44 +882,78 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.AggregateUDF.udaf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">accum</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.AggregateUDF" title="datafusion.udf.AggregateUDF"><span class="pre">AggregateUDF</span></a></span></span><a class="headerlink" href="#datafusion.udf.AggregateUDF.udaf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Aggregate Function.</p> -<p>If your <a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><code class="xref py py-class docutils literal notranslate"><span class="pre">Accumulator</span></code></a> can be instantiated with no arguments, you -can simply pass it’s type as <code class="docutils literal notranslate"><span class="pre">accum</span></code>. If you need to pass additional arguments -to it’s constructor, you can define a lambda or a factory method. During runtime -the <a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><code class="xref py py-class docutils literal notranslate"><span class="pre">Accumulator</span></code></a> will be constructed for every instance in -which this UDAF is used. The following examples are all valid.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> -<span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.compute</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pc</span> - -<span class="k">class</span><span class="w"> </span><span class="nc">Summarize</span><span class="p">(</span><span class="n">Accumulator</span><span class="p">):</span> - <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bias</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">bias</span><span class="p">)</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Scalar</span><span class="p">]:</span> - <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="p">]</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">values</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> <span class="o">+</span> <span class="n">pc</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">values</span><span class="p">)</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">states</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sum</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> <span class="o">+</span> <span class="n">pc</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">states</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">evaluate</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Scalar</span><span class="p">:</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sum</span> - -<span class="k">def</span><span class="w"> </span><span class="nf">sum_bias_10</span><span class="p">()</span> <span class="o">-></span> <span class="n">Summarize</span><span class="p">:</span> - <span class="k">return</span> <span class="n">Summarize</span><span class="p">(</span><span class="mf">10.0</span><span class="p">)</span> - -<span class="n">udaf1</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="n">Summarize</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udaf2</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="n">sum_bias_10</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udaf3</span> <span class="o">=</span> <span class="n">udaf</span><span class="p">(</span><span class="k">lambda</span><span class="p">:</span> <span class="n">Summarize</span><span class="p">(</span><span class="mf">20.0</span><span class="p">),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">(),</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()],</span> <span class="s2">"immutable"</span><span class="p">)</span> -</pre></div> -</div> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.AggregateUDF" title="datafusion.udf.AggregateUDF"><span class="pre">AggregateUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.AggregateUDF.udaf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">accum</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.Accumulator" title="datafusion.udf.Accumulator"><span class="pre">Accumulator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">state_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.AggregateUDF" title="datafusion.udf.AggregateUDF"><span class="pre">AggregateUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Aggregate Function (UDAF).</p> +<p>This class allows you to define an <strong>aggregate function</strong> that can be used in +data aggregation or window function calls.</p> +<dl> +<dt>Usage:</dt><dd><ul class="simple"> +<li><dl class="simple"> +<dt><strong>As a function</strong>: Call <a href="#id1"><span class="problematic" id="id2">`</span></a>udaf(accum, input_types, return_type, state_type,</dt><dd><p>volatility, name)`.</p> +</dd> +</dl> +</li> +<li><dl class="simple"> +<dt><strong>As a decorator</strong>: Use <a href="#id3"><span class="problematic" id="id4">`</span></a>@udaf(input_types, return_type, state_type,</dt><dd><p>volatility, name)`.</p> +</dd> +</dl> +</li> +</ul> +<p>When using <cite>udaf</cite> as a decorator, <strong>do not pass `accum` explicitly</strong>.</p> +</dd> +</dl> +<p><strong>Function example:</strong></p> +<blockquote> +<div><p>If your <cite>:py:class:Accumulator</cite> can be instantiated with no arguments, you +can simply pass it’s type as <cite>accum</cite>. If you need to pass additional +arguments to it’s constructor, you can define a lambda or a factory method. +During runtime the <cite>:py:class:Accumulator</cite> will be constructed for every +instance in which this UDAF is used. The following examples are all valid. +<a href="#id5"><span class="problematic" id="id6">``</span></a>` +import pyarrow as pa +import pyarrow.compute as pc</p> +<dl class="simple"> +<dt>class Summarize(Accumulator):</dt><dd><dl class="simple"> +<dt>def __init__(self, bias: float = 0.0):</dt><dd><p>self._sum = pa.scalar(bias)</p> +</dd> +<dt>def state(self) -> list[pa.Scalar]:</dt><dd><p>return [self._sum]</p> +</dd> +<dt>def update(self, values: pa.Array) -> None:</dt><dd><p>self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py())</p> +</dd> +<dt>def merge(self, states: list[pa.Array]) -> None:</dt><dd><p>self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py())</p> +</dd> +<dt>def evaluate(self) -> pa.Scalar:</dt><dd><p>return self._sum</p> +</dd> +</dl> +</dd> +<dt>def sum_bias_10() -> Summarize:</dt><dd><p>return Summarize(10.0)</p> +</dd> +<dt>udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()],</dt><dd><p>“immutable”)</p> +</dd> +<dt>udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()],</dt><dd><p>“immutable”)</p> +</dd> +<dt>udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),</dt><dd><p>[pa.float64()], “immutable”)</p> +</dd> +</dl> +<p><a href="#id7"><span class="problematic" id="id8">``</span></a><a href="#id9"><span class="problematic" id="id10">`</span></a></p> +</div></blockquote> +<dl> +<dt><strong>Decorator example:</strong></dt><dd><p><a href="#id11"><span class="problematic" id="id12">``</span></a>` +@udaf(pa.float64(), pa.float64(), [pa.float64()], “immutable”) +def udf4() -> Summarize:</p> +<blockquote> +<div><p>return Summarize(10.0)</p> +</div></blockquote> +<p><a href="#id13"><span class="problematic" id="id14">``</span></a><a href="#id15"><span class="problematic" id="id16">`</span></a></p> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>accum</strong> – The accumulator python function.</p></li> +<li><p><strong>accum</strong> – The accumulator python function. <strong>Only needed when calling as a +function. Skip this argument when using `udaf` as a decorator.</strong></p></li> <li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">accum</span></code>.</p></li> <li><p><strong>return_type</strong> – The data type of the return value.</p></li> <li><p><strong>state_type</strong> – The data types of the intermediate accumulation.</p></li> @@ -868,12 +977,12 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.udf.ScalarUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">ScalarUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.ScalarUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">ScalarUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.ScalarUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing scalar user-defined functions (UDF).</p> <p>Scalar UDFs operate on a row by row basis. See also <a class="reference internal" href="#datafusion.udf.AggregateUDF" title="datafusion.udf.AggregateUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">AggregateUDF</span></code></a> for operating on a group of rows.</p> <p>Instantiate a scalar user-defined function (UDF).</p> -<p>See helper method <a class="reference internal" href="#datafusion.udf.ScalarUDF.udf" title="datafusion.udf.ScalarUDF.udf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udf()</span></code></a> for argument details.</p> +<p>See helper method <a class="reference internal" href="#datafusion.udf.udf" title="datafusion.udf.udf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udf()</span></code></a> for argument details.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.ScalarUDF.__call__"> <span class="sig-name descname"><span class="pre">__call__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.udf.ScalarUDF.__call__" title="Link to this definition">¶</a></dt> @@ -884,28 +993,56 @@ <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.ScalarUDF.udf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.ScalarUDF" title="datafusion.udf.ScalarUDF"><span class="pre">ScalarUDF</span></a></span></span><a class="headerlink" href="#datafusion.udf.ScalarUDF.udf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Function.</p> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.ScalarUDF" title="datafusion.udf.ScalarUDF"><span class="pre">ScalarUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.ScalarUDF.udf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">_R</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">_R</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.ScalarUDF" title="datafusion.udf.ScalarUDF"><span class="pre">ScalarUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Function (UDF).</p> +<p>This class can be used both as a <strong>function</strong> and as a <strong>decorator</strong>.</p> +<dl class="simple"> +<dt>Usage:</dt><dd><ul class="simple"> +<li><p><strong>As a function</strong>: Call <cite>udf(func, input_types, return_type, volatility, +name)</cite>.</p></li> +<li><p><strong>As a decorator</strong>: Use <cite>@udf(input_types, return_type, volatility, +name)</cite>. In this case, do <strong>not</strong> pass <cite>func</cite> explicitly.</p></li> +</ul> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>func</strong> – A callable python function.</p></li> -<li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">func</span></code>. This list -must be of the same length as the number of arguments.</p></li> -<li><p><strong>return_type</strong> – The data type of the return value from the python -function.</p></li> -<li><p><strong>volatility</strong> – See <code class="docutils literal notranslate"><span class="pre">Volatility</span></code> for allowed values.</p></li> -<li><p><strong>name</strong> – A descriptive name for the function.</p></li> +<li><p><strong>func</strong> (<em>Callable</em><em>, </em><em>optional</em>) – <strong>Only needed when calling as a function.</strong> +Skip this argument when using <cite>udf</cite> as a decorator.</p></li> +<li><p><strong>input_types</strong> (<em>list</em><em>[</em><em>pa.DataType</em><em>]</em>) – The data types of the arguments +to <cite>func</cite>. This list must be of the same length as the number of +arguments.</p></li> +<li><p><strong>return_type</strong> (<em>_R</em>) – The data type of the return value from the function.</p></li> +<li><p><strong>volatility</strong> (<a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><em>Volatility</em></a><em> | </em><em>str</em>) – See <cite>Volatility</cite> for allowed values.</p></li> +<li><p><strong>name</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em>) – A descriptive name for the function.</p></li> </ul> </dd> <dt class="field-even">Returns<span class="colon">:</span></dt> -<dd class="field-even"><p><dl class="simple"> -<dt>A user-defined aggregate function, which can be used in either data</dt><dd><p>aggregation or window function calls.</p> +<dd class="field-even"><p>A user-defined function that can be used in SQL expressions, +data aggregation, or window function calls.</p> </dd> </dl> -</p> -</dd> -</dl> +<p class="rubric">Example</p> +<p><strong>Using `udf` as a function:</strong> +<a href="#id17"><span class="problematic" id="id18">``</span></a>` +def double_func(x):</p> +<blockquote> +<div><p>return x * 2</p> +</div></blockquote> +<p>double_udf = udf(double_func, [pa.int32()], pa.int32(), +“volatile”, “double_it”) +<a href="#id19"><span class="problematic" id="id20">``</span></a><a href="#id21"><span class="problematic" id="id22">`</span></a></p> +<p><strong>Using `udf` as a decorator:</strong> +<a href="#id23"><span class="problematic" id="id24">``</span></a>` +@udf([pa.int32()], pa.int32(), “volatile”, “double_it”) +def double_udf(x):</p> +<blockquote> +<div><p>return x * 2</p> +</div></blockquote> +<p><a href="#id25"><span class="problematic" id="id26">``</span></a><a href="#id27"><span class="problematic" id="id28">`</span></a></p> </dd></dl> <dl class="py attribute"> @@ -925,7 +1062,7 @@ name (<cite>“immutable”</cite>, <cite>“stable”</cite>, or <cite>“volatile”</cite>).</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.Volatility.__str__"> -<span class="sig-name descname"><span class="pre">__str__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.Volatility.__str__" title="Link to this definition">¶</a></dt> +<span class="sig-name descname"><span class="pre">__str__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.udf.Volatility.__str__" title="Link to this definition">¶</a></dt> <dd><p>Returns the string equivalent.</p> </dd></dl> @@ -1022,9 +1159,12 @@ <dd><p>Evaluate a window function on an entire input partition.</p> <p>This function is called once per input <em>partition</em> for window functions that <em>do not use</em> values from the window frame, such as -<a class="reference internal" href="../functions/index.html#datafusion.functions.row_number" title="datafusion.functions.row_number"><code class="xref py py-func docutils literal notranslate"><span class="pre">row_number()</span></code></a>, <a class="reference internal" href="../functions/index.html#datafusion.functions.rank" title="datafusion.functions.rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">rank()</span></code></a>, -<a class="reference internal" href="../functions/index.html#datafusion.functions.dense_rank" title="datafusion.functions.dense_rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">dense_rank()</span></code></a>, <a class="reference internal" href="../functions/index.html#datafusion.functions.percent_rank" title="datafusion.functions.percent_rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">percent_rank()</span></code></a>, -<a class="reference internal" href="../functions/index.html#datafusion.functions.cume_dist" title="datafusion.functions.cume_dist"><code class="xref py py-func docutils literal notranslate"><span class="pre">cume_dist()</span></code></a>, <a class="reference internal" href="../functions/index.html#datafusion.functions.lead" title="datafusion.functions.lead"><code class="xref py py-func docutils literal notranslate"><span class="pre">lead()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.row_number" title="datafusion.functions.row_number"><code class="xref py py-func docutils literal notranslate"><span class="pre">row_number()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.rank" title="datafusion.functions.rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">rank()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.dense_rank" title="datafusion.functions.dense_rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">dense_rank()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.percent_rank" title="datafusion.functions.percent_rank"><code class="xref py py-func docutils literal notranslate"><span class="pre">percent_rank()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.cume_dist" title="datafusion.functions.cume_dist"><code class="xref py py-func docutils literal notranslate"><span class="pre">cume_dist()</span></code></a>, +<a class="reference internal" href="../functions/index.html#datafusion.functions.lead" title="datafusion.functions.lead"><code class="xref py py-func docutils literal notranslate"><span class="pre">lead()</span></code></a>, and <a class="reference internal" href="../functions/index.html#datafusion.functions.lag" title="datafusion.functions.lag"><code class="xref py py-func docutils literal notranslate"><span class="pre">lag()</span></code></a>.</p> <p>It produces the result of all rows in a single pass. It expects to receive the entire partition as the <code class="docutils literal notranslate"><span class="pre">value</span></code> and @@ -1139,12 +1279,12 @@ <dl class="py class"> <dt class="sig sig-object py" id="datafusion.udf.WindowUDF"> -<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">WindowUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.WindowUDF" title="Link to this definition">¶</a></dt> +<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">WindowUDF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.udf.WindowUDF" title="Link to this definition">¶</a></dt> <dd><p>Class for performing window user-defined functions (UDF).</p> <p>Window UDFs operate on a partition of rows. See also <a class="reference internal" href="#datafusion.udf.ScalarUDF" title="datafusion.udf.ScalarUDF"><code class="xref py py-class docutils literal notranslate"><span class="pre">ScalarUDF</span></code></a> for operating on a row by row basis.</p> <p>Instantiate a user-defined window function (UDWF).</p> -<p>See <a class="reference internal" href="#datafusion.udf.WindowUDF.udwf" title="datafusion.udf.WindowUDF.udwf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udwf()</span></code></a> for a convenience function and argument +<p>See <a class="reference internal" href="#datafusion.udf.udwf" title="datafusion.udf.udwf"><code class="xref py py-func docutils literal notranslate"><span class="pre">udwf()</span></code></a> for a convenience function and argument descriptions.</p> <dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.WindowUDF.__call__"> @@ -1155,44 +1295,86 @@ </dd></dl> <dl class="py method"> +<dt class="sig sig-object py" id="datafusion.udf.WindowUDF._create_window_udf"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_create_window_udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.WindowUDF" title="datafusion.udf.WindowUDF"><span class="pre">WindowUDF</span></a></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF._create_window_udf" title="Link to this definition">¶</a></dt> +<dd><p>Create a WindowUDF instance from function arguments.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.udf.WindowUDF._create_window_udf_decorator"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_create_window_udf_decorator</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF._create_window_udf_decorator" title="Link to this definition">¶</a></dt> +<dd><p>Create a decorator for a WindowUDF.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.udf.WindowUDF._get_default_name"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_get_default_name</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF._get_default_name" title="Link to this definition">¶</a></dt> +<dd><p>Get the default name for a function based on its attributes.</p> +</dd></dl> + +<dl class="py method"> +<dt class="sig sig-object py" id="datafusion.udf.WindowUDF._normalize_input_types"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_normalize_input_types</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF._normalize_input_types" title="Link to this definition">¶</a></dt> +<dd><p>Convert a single DataType to a list if needed.</p> +</dd></dl> + +<dl class="py method"> <dt class="sig sig-object py" id="datafusion.udf.WindowUDF.udwf"> -<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.WindowUDF" title="datafusion.udf.WindowUDF"><span class="pre">WindowUDF</span></a></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF.udwf" title="Link to this definition">¶</a></dt> -<dd><p>Create a new User-Defined Window Function.</p> -<p>If your <a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><code class="xref py py-class docutils literal notranslate"><span class="pre">WindowEvaluator</span></code></a> can be instantiated with no arguments, you -can simply pass it’s type as <code class="docutils literal notranslate"><span class="pre">func</span></code>. If you need to pass additional arguments -to it’s constructor, you can define a lambda or a factory method. During runtime -the <a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><code class="xref py py-class docutils literal notranslate"><span class="pre">WindowEvaluator</span></code></a> will be constructed for every instance in -which this UDWF is used. The following examples are all valid.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> - -<span class="k">class</span><span class="w"> </span><span class="nc">BiasedNumbers</span><span class="p">(</span><span class="n">WindowEvaluator</span><span class="p">):</span> - <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">start</span> <span class="o">=</span> <span class="n">start</span> - - <span class="k">def</span><span class="w"> </span><span class="nf">evaluate_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">values</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">],</span> <span class="n">num_rows</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span> - <span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span> <span class="o">+</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_rows</span><span class="p">)])</span> - -<span class="k">def</span><span class="w"> </span><span class="nf">bias_10</span><span class="p">()</span> <span class="o">-></span> <span class="n">BiasedNumbers</span><span class="p">:</span> - <span class="k">return</span> <span class="n">BiasedNumbers</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> - -<span class="n">udwf1</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="n">BiasedNumbers</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udwf2</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="n">bias_10</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -<span class="n">udwf3</span> <span class="o">=</span> <span class="n">udwf</span><span class="p">(</span><span class="k">lambda</span><span class="p">:</span> <span class="n">BiasedNumbers</span><span class="p">(</span><span class="mi">20</span><span class="p">),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="s2">"immutable"</span><span class="p">)</span> -</pre></div> -</div> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowUDF" title="datafusion.udf.WindowUDF"><span class="pre">WindowUDF</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.udf.WindowUDF.udwf" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py"> +<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Callable</span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.udf.WindowEvaluator" title="datafusion.udf.WindowEvaluator"><span class="pre">WindowEvaluator</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_types</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_type</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.DataType</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">volatility</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><span class="pre">Volatility</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.udf.WindowUDF" title="datafusion.udf.WindowUDF"><span class="pre">WindowUDF</span></a></span></span></dt> +<dd><p>Create a new User-Defined Window Function (UDWF).</p> +<p>This class can be used both as a <strong>function</strong> and as a <strong>decorator</strong>.</p> +<dl> +<dt>Usage:</dt><dd><ul class="simple"> +<li><p><strong>As a function</strong>: Call <cite>udwf(func, input_types, return_type, volatility, +name)</cite>.</p></li> +<li><p><strong>As a decorator</strong>: Use <cite>@udwf(input_types, return_type, volatility, +name)</cite>. When using <cite>udwf</cite> as a decorator, <strong>do not pass `func` +explicitly</strong>.</p></li> +</ul> +</dd> +<dt><strong>Function example:</strong></dt><dd><p><a href="#id29"><span class="problematic" id="id30">``</span></a>` +import pyarrow as pa</p> +<dl class="simple"> +<dt>class BiasedNumbers(WindowEvaluator):</dt><dd><dl class="simple"> +<dt>def __init__(self, start: int = 0) -> None:</dt><dd><p>self.start = start</p> +</dd> +<dt>def evaluate_all(self, values: list[pa.Array],</dt><dd><p>num_rows: int) -> pa.Array: +return pa.array([self.start + i for i in range(num_rows)])</p> +</dd> +</dl> +</dd> +<dt>def bias_10() -> BiasedNumbers:</dt><dd><p>return BiasedNumbers(10)</p> +</dd> +</dl> +<p>udwf1 = udwf(BiasedNumbers, pa.int64(), pa.int64(), “immutable”) +udwf2 = udwf(bias_10, pa.int64(), pa.int64(), “immutable”) +udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), “immutable”)</p> +<p><a href="#id31"><span class="problematic" id="id32">``</span></a><a href="#id33"><span class="problematic" id="id34">`</span></a></p> +</dd> +<dt><strong>Decorator example:</strong></dt><dd><p><a href="#id35"><span class="problematic" id="id36">``</span></a>` +@udwf(pa.int64(), pa.int64(), “immutable”) +def biased_numbers() -> BiasedNumbers:</p> +<blockquote> +<div><p>return BiasedNumbers(10)</p> +</div></blockquote> +<p><a href="#id37"><span class="problematic" id="id38">``</span></a><a href="#id39"><span class="problematic" id="id40">`</span></a></p> +</dd> +</dl> <dl class="field-list simple"> <dt class="field-odd">Parameters<span class="colon">:</span></dt> <dd class="field-odd"><ul class="simple"> -<li><p><strong>func</strong> – A callable to create the window function.</p></li> -<li><p><strong>input_types</strong> – The data types of the arguments to <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li> +<li><p><strong>func</strong> – <strong>Only needed when calling as a function. Skip this argument when +using `udwf` as a decorator.</strong></p></li> +<li><p><strong>input_types</strong> – The data types of the arguments.</p></li> <li><p><strong>return_type</strong> – The data type of the return value.</p></li> <li><p><strong>volatility</strong> – See <a class="reference internal" href="#datafusion.udf.Volatility" title="datafusion.udf.Volatility"><code class="xref py py-class docutils literal notranslate"><span class="pre">Volatility</span></code></a> for allowed values.</p></li> -<li><p><strong>arguments</strong> – A list of arguments to pass in to the __init__ method for accum.</p></li> <li><p><strong>name</strong> – A descriptive name for the function.</p></li> </ul> </dd> <dt class="field-even">Returns<span class="colon">:</span></dt> -<dd class="field-even"><p>A user-defined window function.</p> +<dd class="field-even"><p>A user-defined window function that can be used in window function calls.</p> </dd> </dl> </dd></dl> @@ -1210,8 +1392,18 @@ <dd></dd></dl> <dl class="py data"> -<dt class="sig sig-object py" id="datafusion.udf._W"> -<span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">_W</span></span><a class="headerlink" href="#datafusion.udf._W" title="Link to this definition">¶</a></dt> +<dt class="sig sig-object py" id="datafusion.udf.udaf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">udaf</span></span><a class="headerlink" href="#datafusion.udf.udaf" title="Link to this definition">¶</a></dt> +<dd></dd></dl> + +<dl class="py data"> +<dt class="sig sig-object py" id="datafusion.udf.udf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">udf</span></span><a class="headerlink" href="#datafusion.udf.udf" title="Link to this definition">¶</a></dt> +<dd></dd></dl> + +<dl class="py data"> +<dt class="sig sig-object py" id="datafusion.udf.udwf"> +<span class="sig-prename descclassname"><span class="pre">datafusion.udf.</span></span><span class="sig-name descname"><span class="pre">udwf</span></span><a class="headerlink" href="#datafusion.udf.udwf" title="Link to this definition">¶</a></dt> <dd></dd></dl> </section>
diff --git a/autoapi/index.html b/autoapi/index.html index da449fc..40c2c4c 100644 --- a/autoapi/index.html +++ b/autoapi/index.html
@@ -35,7 +35,7 @@ <link rel="index" title="Index" href="../genindex.html" /> <link rel="search" title="Search" href="../search.html" /> <link rel="next" title="datafusion" href="datafusion/index.html" /> - <link rel="prev" title="Introduction" href="../contributor-guide/introduction.html" /> + <link rel="prev" title="Python Extensions" href="../contributor-guide/ffi.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="datafusion/object_store/index.html"> datafusion.object_store </a> @@ -392,6 +407,7 @@ <li class="toctree-l3"><a class="reference internal" href="datafusion/input/location/index.html">datafusion.input.location</a></li> </ul> </li> +<li class="toctree-l2"><a class="reference internal" href="datafusion/io/index.html">datafusion.io</a></li> <li class="toctree-l2"><a class="reference internal" href="datafusion/object_store/index.html">datafusion.object_store</a></li> <li class="toctree-l2"><a class="reference internal" href="datafusion/plan/index.html">datafusion.plan</a></li> <li class="toctree-l2"><a class="reference internal" href="datafusion/record_batch/index.html">datafusion.record_batch</a></li> @@ -415,11 +431,11 @@ <!-- Previous / next buttons --> <div class='prev-next-area'> - <a class='left-prev' id="prev-link" href="../contributor-guide/introduction.html" title="previous page"> + <a class='left-prev' id="prev-link" href="../contributor-guide/ffi.html" title="previous page"> <i class="fas fa-angle-left"></i> <div class="prev-next-info"> <p class="prev-next-subtitle">previous</p> - <p class="prev-next-title">Introduction</p> + <p class="prev-next-title">Python Extensions</p> </div> </a> <a class='right-next' id="next-link" href="datafusion/index.html" title="next page">
diff --git a/contributor-guide/ffi.html b/contributor-guide/ffi.html new file mode 100644 index 0000000..7abc766 --- /dev/null +++ b/contributor-guide/ffi.html
@@ -0,0 +1,649 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> + + <title>Python Extensions — Apache Arrow DataFusion documentation</title> + + <link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> +<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> + + + <link rel="stylesheet" + href="../_static/vendor/fontawesome/5.13.0/css/all.min.css"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2"> + + + + + + <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" /> + <link rel="stylesheet" type="text/css" href="../_static/styles/pydata-sphinx-theme.css?v=1140d252" /> + <link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" /> + <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=dca7052a" /> + + <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"> + + <script src="../_static/documentation_options.js?v=8a448e45"></script> + <script src="../_static/doctools.js?v=9bcbadda"></script> + <script src="../_static/sphinx_highlight.js?v=dc90522c"></script> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="API Reference" href="../autoapi/index.html" /> + <link rel="prev" title="Introduction" href="introduction.html" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <meta name="docsearch:language" content="en"> + + + <!-- Google Analytics --> + + </head> + <body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80"> + + <div class="container-fluid" id="banner"></div> + + + + + <div class="container-xl"> + <div class="row"> + + + <!-- Only show if we have sidebars configured, else just a small margin --> + <div class="col-12 col-md-3 bd-sidebar"> + <div class="sidebar-start-items"> +<a class="navbar-brand" href="../index.html"> + <img src="../_static/images/2x_bgwhite_original.png" class="logo" alt="logo"> +</a> + +<form class="bd-search d-flex align-items-center" action="../search.html" method="get"> + <i class="icon fas fa-search"></i> + <input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" > +</form> + +<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation"> + <div class="bd-toc-item active"> + + <p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + LINKS + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python"> + Github and Issue Tracker + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/"> + Rust's API Docs + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md"> + Code of conduct + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples"> + Examples + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + USER GUIDE + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="../user-guide/introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../user-guide/basics.html"> + Concepts + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../user-guide/data-sources.html"> + Data Sources + </a> + </li> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../user-guide/common-operations/index.html"> + Common Operations + </a> + <input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/> + <label for="toctree-checkbox-1"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/basic-info.html"> + Basic Operations + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/select-and-filter.html"> + Column Selections + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/expressions.html"> + Expressions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/joins.html"> + Joins + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/functions.html"> + Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/aggregations.html"> + Aggregation + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/windows.html"> + Window Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html"> + User-Defined Functions + </a> + </li> + </ul> + </li> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../user-guide/io/index.html"> + IO + </a> + <input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/> + <label for="toctree-checkbox-2"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/arrow.html"> + Arrow + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/avro.html"> + Avro + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/csv.html"> + CSV + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/json.html"> + JSON + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/parquet.html"> + Parquet + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/io/table_provider.html"> + Custom Table Provider + </a> + </li> + </ul> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../user-guide/configuration.html"> + Configuration + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../user-guide/sql.html"> + SQL + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + CONTRIBUTOR GUIDE + </span> +</p> +<ul class="current nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1 current active"> + <a class="current reference internal" href="#"> + Python Extensions + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + API + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../autoapi/index.html"> + API Reference + </a> + <input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/> + <label for="toctree-checkbox-3"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2 has-children"> + <a class="reference internal" href="../autoapi/datafusion/index.html"> + datafusion + </a> + <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/> + <label for="toctree-checkbox-4"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/catalog/index.html"> + datafusion.catalog + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/context/index.html"> + datafusion.context + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/dataframe/index.html"> + datafusion.dataframe + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/expr/index.html"> + datafusion.expr + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/functions/index.html"> + datafusion.functions + </a> + </li> + <li class="toctree-l3 has-children"> + <a class="reference internal" href="../autoapi/datafusion/input/index.html"> + datafusion.input + </a> + <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/> + <label for="toctree-checkbox-5"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l4"> + <a class="reference internal" href="../autoapi/datafusion/input/base/index.html"> + datafusion.input.base + </a> + </li> + <li class="toctree-l4"> + <a class="reference internal" href="../autoapi/datafusion/input/location/index.html"> + datafusion.input.location + </a> + </li> + </ul> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> + datafusion.object_store + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/plan/index.html"> + datafusion.plan + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/record_batch/index.html"> + datafusion.record_batch + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/substrait/index.html"> + datafusion.substrait + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/udf/index.html"> + datafusion.udf + </a> + </li> + </ul> + </li> + </ul> + </li> +</ul> + + + </div> +</nav> + </div> + <div class="sidebar-end-items"> + </div> + </div> + + + + + <div class="d-none d-xl-block col-xl-2 bd-toc"> + + + <div class="toc-item"> + +<div class="tocsection onthispage pt-5 pb-3"> + <i class="fas fa-list"></i> On this page +</div> + +<nav id="bd-toc-nav"> + <ul class="visible nav section-nav flex-column"> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#the-primary-issue"> + The Primary Issue + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#the-ffi-approach"> + The FFI Approach + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#inspiration-from-arrow"> + Inspiration from Arrow + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#implementation-details"> + Implementation Details + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#alternative-approach"> + Alternative Approach + </a> + </li> + <li class="toc-h2 nav-item toc-entry"> + <a class="reference internal nav-link" href="#status-of-work"> + Status of Work + </a> + </li> +</ul> + +</nav> + </div> + + <div class="toc-item"> + + </div> + + + </div> + + + + + + + <main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main"> + + <div> + + <section id="python-extensions"> +<h1>Python Extensions<a class="headerlink" href="#python-extensions" title="Link to this heading">¶</a></h1> +<p>The DataFusion in Python project is designed to allow users to extend its functionality in a few core +areas. Ideally many users would like to package their extensions as a Python package and easily +integrate that package with this project. This page serves to describe some of the challenges we face +when doing these integrations and the approach our project uses.</p> +<section id="the-primary-issue"> +<h2>The Primary Issue<a class="headerlink" href="#the-primary-issue" title="Link to this heading">¶</a></h2> +<p>Suppose you wish to use DataFusion and you have a custom data source that can produce tables that +can then be queried against, similar to how you can register a <a class="reference internal" href="../user-guide/io/csv.html#io-csv"><span class="std std-ref">CSV</span></a> or +<a class="reference internal" href="../user-guide/io/parquet.html#io-parquet"><span class="std std-ref">Parquet</span></a> file. In DataFusion terminology, you likely want to implement a +<a class="reference internal" href="../user-guide/io/table_provider.html#io-custom-table-provider"><span class="std std-ref">Custom Table Provider</span></a>. In an effort to make your data source +as performant as possible and to utilize the features of DataFusion, you may decide to write +your source in Rust and then expose it through <a class="reference external" href="https://pyo3.rs">PyO3</a> as a Python library.</p> +<p>At first glance, it may appear the best way to do this is to add the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> +crate as a dependency, provide a <code class="docutils literal notranslate"><span class="pre">PyTable</span></code>, and then to register it with the +<code class="docutils literal notranslate"><span class="pre">SessionContext</span></code>. Unfortunately, this will not work.</p> +<p>When you produce your code as a Python library and it needs to interact with the DataFusion +library, at the lowest level they communicate through an Application Binary Interface (ABI). +The acronym sounds similar to API (Application Programming Interface), but it is distinctly +different.</p> +<p>The ABI sets the standard for how these libraries can share data and functions between each +other. One of the key differences between Rust and other programming languages is that Rust +does not have a stable ABI. What this means in practice is that if you compile a Rust library +with one version of the <code class="docutils literal notranslate"><span class="pre">rustc</span></code> compiler and I compile another library to interface with it +but I use a different version of the compiler, there is no guarantee the interface will be +the same.</p> +<p>In practice, this means that a Python library built with <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> as a Rust +dependency will generally <strong>not</strong> be compatible with the DataFusion Python package, even +if they reference the same version of <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code>. If you attempt to do this, it may +work on your local computer if you have built both packages with the same optimizations. +This can sometimes lead to a false expectation that the code will work, but it frequently +breaks the moment you try to use your package against the released packages.</p> +<p>You can find more information about the Rust ABI in their +<a class="reference external" href="https://doc.rust-lang.org/reference/abi.html">online documentation</a>.</p> +</section> +<section id="the-ffi-approach"> +<h2>The FFI Approach<a class="headerlink" href="#the-ffi-approach" title="Link to this heading">¶</a></h2> +<p>Rust supports interacting with other programming languages through it’s Foreign Function +Interface (FFI). The advantage of using the FFI is that it enables you to write data structures +and functions that have a stable ABI. The allows you to use Rust code with C, Python, and +other languages. In fact, the <a class="reference external" href="https://pyo3.rs">PyO3</a> library uses the FFI to share data +and functions between Python and Rust.</p> +<p>The approach we are taking in the DataFusion in Python project is to incrementally expose +more portions of the DataFusion project via FFI interfaces. This allows users to write Rust +code that does <strong>not</strong> require the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> crate as a dependency, expose their +code in Python via PyO3, and have it interact with the DataFusion Python package.</p> +<p>Early adopters of this approach include <a class="reference external" href="https://delta-io.github.io/delta-rs/">delta-rs</a> +who has adapted their Table Provider for use in <code class="docutils literal notranslate"><span class="pre">`datafusion-python`</span></code> with only a few lines +of code. Also, the DataFusion Python project uses the existing definitions from +<a class="reference external" href="https://arrow.apache.org/docs/format/CStreamInterface.html">Apache Arrow CStream Interface</a> +to support importing <strong>and</strong> exporting tables. Any Python package that supports reading +the Arrow C Stream interface can work with DataFusion Python out of the box! You can read +more about working with Arrow sources in the <a class="reference internal" href="../user-guide/data-sources.html#user-guide-data-sources"><span class="std std-ref">Data Sources</span></a> +page.</p> +<p>To learn more about the Foreign Function Interface in Rust, the +<a class="reference external" href="https://doc.rust-lang.org/nomicon/ffi.html">Rustonomicon</a> is a good resource.</p> +</section> +<section id="inspiration-from-arrow"> +<h2>Inspiration from Arrow<a class="headerlink" href="#inspiration-from-arrow" title="Link to this heading">¶</a></h2> +<p>DataFusion is built upon <a class="reference external" href="https://arrow.apache.org/">Apache Arrow</a>. The canonical Python +Arrow implementation, <a class="reference external" href="https://arrow.apache.org/docs/python/index.html">pyarrow</a> provides +an excellent way to share Arrow data between Python projects without performing any copy +operations on the data. They do this by using a well defined set of interfaces. You can +find the details about their stream interface +<a class="reference external" href="https://arrow.apache.org/docs/format/CStreamInterface.html">here</a>. The +<a class="reference external" href="https://github.com/apache/arrow-rs">Rust Arrow Implementation</a> also supports these +<code class="docutils literal notranslate"><span class="pre">C</span></code> style definitions via the Foreign Function Interface.</p> +<p>In addition to using these interfaces to transfer Arrow data between libraries, <code class="docutils literal notranslate"><span class="pre">pyarrow</span></code> +goes one step further to make sharing the interfaces easier in Python. They do this +by exposing PyCapsules that contain the expected functionality.</p> +<p>You can learn more about PyCapsules from the official +<a class="reference external" href="https://docs.python.org/3/c-api/capsule.html">Python online documentation</a>. PyCapsules +have excellent support in PyO3 already. The +<a class="reference external" href="https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule">PyO3 online documentation</a> is a good source +for more details on using PyCapsules in Rust.</p> +<p>Two lessons we leverage from the Arrow project in DataFusion Python are:</p> +<ul class="simple"> +<li><p>We reuse the existing Arrow FFI functionality wherever possible.</p></li> +<li><p>We expose PyCapsules that contain a FFI stable struct.</p></li> +</ul> +</section> +<section id="implementation-details"> +<h2>Implementation Details<a class="headerlink" href="#implementation-details" title="Link to this heading">¶</a></h2> +<p>The bulk of the code necessary to perform our FFI operations is in the upstream +<a class="reference external" href="https://datafusion.apache.org/">DataFusion</a> core repository. You can review the code and +documentation in the <a class="reference external" href="https://crates.io/crates/datafusion-ffi">datafusion-ffi</a> crate.</p> +<p>Our FFI implementation is narrowly focused at sharing data and functions with Rust backed +libraries. This allows us to use the <a class="reference external" href="https://crates.io/crates/abi_stable">abi_stable crate</a>. +This is an excellent crate that allows for easy conversion between Rust native types +and FFI-safe alternatives. For example, if you needed to pass a <code class="docutils literal notranslate"><span class="pre">Vec<String></span></code> via FFI, +you can simply convert it to a <code class="docutils literal notranslate"><span class="pre">RVec<RString></span></code> in an intuitive manner. It also supports +features like <code class="docutils literal notranslate"><span class="pre">RResult</span></code> and <code class="docutils literal notranslate"><span class="pre">ROption</span></code> that do not have an obvious translation to a +C equivalent.</p> +<p>The <a class="reference external" href="https://crates.io/crates/datafusion-ffi">datafusion-ffi</a> crate has been designed to make it easy to convert from DataFusion +traits into their FFI counterparts. For example, if you have defined a custom +<a class="reference external" href="https://docs.rs/datafusion/45.0.0/datafusion/catalog/trait.TableProvider.html">TableProvider</a> +and you want to create a sharable FFI counterpart, you could write:</p> +<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">my_provider</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">MyTableProvider</span><span class="p">::</span><span class="n">default</span><span class="p">();</span> +<span class="kd">let</span><span class="w"> </span><span class="n">ffi_provider</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">FFI_TableProvider</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">Arc</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">my_provider</span><span class="p">),</span><span class="w"> </span><span class="kc">false</span><span class="p">,</span><span class="w"> </span><span class="nb">None</span><span class="p">);</span> +</pre></div> +</div> +<p>If you were interfacing with a library that provided the above <code class="docutils literal notranslate"><span class="pre">FFI_TableProvider</span></code> and +you needed to turn it back into an <code class="docutils literal notranslate"><span class="pre">TableProvider</span></code>, you can turn it into a +<code class="docutils literal notranslate"><span class="pre">ForeignTableProvider</span></code> with implements the <code class="docutils literal notranslate"><span class="pre">TableProvider</span></code> trait.</p> +<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">foreign_provider</span><span class="p">:</span><span class="w"> </span><span class="nc">ForeignTableProvider</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ffi_provider</span><span class="p">.</span><span class="n">into</span><span class="p">();</span> +</pre></div> +</div> +<p>If you review the code in <a class="reference external" href="https://crates.io/crates/datafusion-ffi">datafusion-ffi</a> you will find that each of the traits we share +across the boundary has two portions, one with a <code class="docutils literal notranslate"><span class="pre">FFI_</span></code> prefix and one with a <code class="docutils literal notranslate"><span class="pre">Foreign</span></code> +prefix. This is used to distinguish which side of the FFI boundary that struct is +designed to be used on. The structures with the <code class="docutils literal notranslate"><span class="pre">FFI_</span></code> prefix are to be used on the +<strong>provider</strong> of the structure. In the example we’re showing, this means the code that has +written the underlying <code class="docutils literal notranslate"><span class="pre">TableProvider</span></code> implementation to access your custom data source. +The structures with the <code class="docutils literal notranslate"><span class="pre">Foreign</span></code> prefix are to be used by the receiver. In this case, +it is the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> library.</p> +<p>In order to share these FFI structures, we need to wrap them in some kind of Python object +that can be used to interface from one package to another. As described in the above +section on our inspiration from Arrow, we use <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code>. We can create a <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code> +for our provider thusly:</p> +<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">CString</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="s">"datafusion_table_provider"</span><span class="p">)</span><span class="o">?</span><span class="p">;</span> +<span class="kd">let</span><span class="w"> </span><span class="n">my_capsule</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">PyCapsule</span><span class="p">::</span><span class="n">new_bound</span><span class="p">(</span><span class="n">py</span><span class="p">,</span><span class="w"> </span><span class="n">provider</span><span class="p">,</span><span class="w"> </span><span class="nb">Some</span><span class="p">(</span><span class="n">name</span><span class="p">))</span><span class="o">?</span><span class="p">;</span> +</pre></div> +</div> +<p>On the receiving side, turn this pycapsule object into the <code class="docutils literal notranslate"><span class="pre">FFI_TableProvider</span></code>, which +can then be turned into a <code class="docutils literal notranslate"><span class="pre">ForeignTableProvider</span></code> the associated code is:</p> +<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="kd">let</span><span class="w"> </span><span class="n">capsule</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">capsule</span><span class="p">.</span><span class="n">downcast</span><span class="p">::</span><span class="o"><</span><span class="n">PyCapsule</span><span class="o">></span><span class="p">()</span><span class="o">?</span><span class="p">;</span> +<span class="kd">let</span><span class="w"> </span><span class="n">provider</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">unsafe</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">capsule</span><span class="p">.</span><span class="n">reference</span><span class="p">::</span><span class="o"><</span><span class="n">FFI_TableProvider</span><span class="o">></span><span class="p">()</span><span class="w"> </span><span class="p">};</span> +</pre></div> +</div> +<p>By convention the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> library expects a Python object that has a +<code class="docutils literal notranslate"><span class="pre">TableProvider</span></code> PyCapsule to have this capsule accessible by calling a function named +<code class="docutils literal notranslate"><span class="pre">__datafusion_table_provider__</span></code>. You can see a complete working example of how to +share a <code class="docutils literal notranslate"><span class="pre">TableProvider</span></code> from one python library to DataFusion Python in the +<a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples/ffi-table-provider">repository examples folder</a>.</p> +<p>This section has been written using <code class="docutils literal notranslate"><span class="pre">TableProvider</span></code> as an example. It is the first +extension that has been written using this approach and the most thoroughly implemented. +As we continue to expose more of the DataFusion features, we intend to follow this same +design pattern.</p> +</section> +<section id="alternative-approach"> +<h2>Alternative Approach<a class="headerlink" href="#alternative-approach" title="Link to this heading">¶</a></h2> +<p>Suppose you needed to expose some other features of DataFusion and you could not wait +for the upstream repository to implement the FFI approach we describe. In this case +you decide to create your dependency on the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> crate instead.</p> +<p>As we discussed, this is not guaranteed to work across different compiler versions and +optimization levels. If you wish to go down this route, there are two approaches we +have identified you can use.</p> +<ol class="arabic simple"> +<li><p>Re-export all of <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> yourself with your extensions built in.</p></li> +<li><p>Carefully synchonize your software releases with the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> CI build +system so that your libraries use the exact same compiler, features, and +optimization level.</p></li> +</ol> +<p>We currently do not recommend either of these approaches as they are difficult to +maintain over a long period. Additionally, they require a tight version coupling +between libraries.</p> +</section> +<section id="status-of-work"> +<h2>Status of Work<a class="headerlink" href="#status-of-work" title="Link to this heading">¶</a></h2> +<p>At the time of this writing, the FFI features are under active development. To see +the latest status, we recommend reviewing the code in the <a class="reference external" href="https://crates.io/crates/datafusion-ffi">datafusion-ffi</a> crate.</p> +</section> +</section> + + + </div> + + + <!-- Previous / next buttons --> +<div class='prev-next-area'> + <a class='left-prev' id="prev-link" href="introduction.html" title="previous page"> + <i class="fas fa-angle-left"></i> + <div class="prev-next-info"> + <p class="prev-next-subtitle">previous</p> + <p class="prev-next-title">Introduction</p> + </div> + </a> + <a class='right-next' id="next-link" href="../autoapi/index.html" title="next page"> + <div class="prev-next-info"> + <p class="prev-next-subtitle">next</p> + <p class="prev-next-title">API Reference</p> + </div> + <i class="fas fa-angle-right"></i> + </a> +</div> + + </main> + + + </div> + </div> + + <script src="../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script> + +<!-- Based on pydata_sphinx_theme/footer.html --> +<footer class="footer mt-5 mt-md-0"> + <div class="container"> + + <div class="footer-item"> + <p class="copyright"> + © Copyright 2019-2024, Apache Software Foundation.<br> +</p> + </div> + + <div class="footer-item"> + <p class="sphinx-version"> +Created using <a href="http://sphinx-doc.org/">Sphinx</a> 8.1.3.<br> +</p> + </div> + + <div class="footer-item"> + <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> + <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> + </div> + </div> +</footer> + + + </body> +</html> \ No newline at end of file
diff --git a/contributor-guide/introduction.html b/contributor-guide/introduction.html index 1ff78ce..6a97a45 100644 --- a/contributor-guide/introduction.html +++ b/contributor-guide/introduction.html
@@ -34,7 +34,7 @@ <script src="../_static/sphinx_highlight.js?v=dc90522c"></script> <link rel="index" title="Index" href="../genindex.html" /> <link rel="search" title="Search" href="../search.html" /> - <link rel="next" title="API Reference" href="../autoapi/index.html" /> + <link rel="next" title="Python Extensions" href="ffi.html" /> <link rel="prev" title="SQL" href="../user-guide/sql.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -484,7 +499,7 @@ a file. If you place this file in your project directory with filename <code class="docutils literal notranslate"><span class="pre">.pyo3_build_config</span></code> it will be ignored by <code class="docutils literal notranslate"><span class="pre">git</span></code>.</p> <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">implementation</span><span class="o">=</span><span class="n">CPython</span> -<span class="n">version</span><span class="o">=</span><span class="mf">3.8</span> +<span class="n">version</span><span class="o">=</span><span class="mf">3.9</span> <span class="n">shared</span><span class="o">=</span><span class="n">true</span> <span class="n">abi3</span><span class="o">=</span><span class="n">true</span> <span class="n">lib_name</span><span class="o">=</span><span class="n">python3</span><span class="mf">.12</span> @@ -528,10 +543,10 @@ <p class="prev-next-title">SQL</p> </div> </a> - <a class='right-next' id="next-link" href="../autoapi/index.html" title="next page"> + <a class='right-next' id="next-link" href="ffi.html" title="next page"> <div class="prev-next-info"> <p class="prev-next-subtitle">next</p> - <p class="prev-next-title">API Reference</p> + <p class="prev-next-title">Python Extensions</p> </div> <i class="fas fa-angle-right"></i> </a>
diff --git a/genindex.html b/genindex.html index 6d7fa3c..6d65db2 100644 --- a/genindex.html +++ b/genindex.html
@@ -125,6 +125,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -230,6 +235,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -305,6 +315,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -526,8 +541,6 @@ <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.__radd__">(datafusion.expr.Expr attribute)</a> </li> </ul></li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/index.html#datafusion.Expr.__rand__">__rand__ (datafusion.Expr attribute)</a> <ul> @@ -552,6 +565,8 @@ <li><a href="autoapi/datafusion/plan/index.html#datafusion.plan.LogicalPlan.__repr__">(datafusion.plan.LogicalPlan method)</a> </li> </ul></li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/index.html#datafusion.Expr.__richcmp__">__richcmp__() (datafusion.Expr method)</a> <ul> @@ -602,6 +617,30 @@ <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.__truediv__">(datafusion.expr.Expr method)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowUDF._create_window_udf">_create_window_udf() (datafusion.udf.WindowUDF static method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF._create_window_udf">(datafusion.WindowUDF static method)</a> +</li> + </ul></li> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowUDF._create_window_udf_decorator">_create_window_udf_decorator() (datafusion.udf.WindowUDF static method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF._create_window_udf_decorator">(datafusion.WindowUDF static method)</a> +</li> + </ul></li> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowUDF._get_default_name">_get_default_name() (datafusion.udf.WindowUDF static method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF._get_default_name">(datafusion.WindowUDF static method)</a> +</li> + </ul></li> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowUDF._normalize_input_types">_normalize_input_types() (datafusion.udf.WindowUDF static method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF._normalize_input_types">(datafusion.WindowUDF static method)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Window._null_treatment">_null_treatment (datafusion.expr.Window attribute)</a> </li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Window._order_by">_order_by (datafusion.expr.Window attribute)</a> @@ -646,8 +685,6 @@ <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF._udwf">(datafusion.WindowUDF attribute)</a> </li> </ul></li> - <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf._W">_W (in module datafusion.udf)</a> -</li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Window._window_frame">_window_frame (datafusion.expr.Window attribute)</a> </li> </ul></td> @@ -826,10 +863,10 @@ </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_length">bit_length() (in module datafusion.functions)</a> </li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_or">bit_or() (in module datafusion.functions)</a> </li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_xor">bit_xor() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bool_and">bool_and() (in module datafusion.functions)</a> @@ -845,6 +882,8 @@ <ul> <li><a href="autoapi/datafusion/input/location/index.html#datafusion.input.location.LocationInputPlugin.build_table">(datafusion.input.location.LocationInputPlugin method)</a> </li> + <li><a href="autoapi/datafusion/input/index.html#datafusion.input.LocationInputPlugin.build_table">(datafusion.input.LocationInputPlugin method)</a> +</li> </ul></li> </ul></td> </tr></table> @@ -1080,6 +1119,13 @@ </li> </ul></li> <li> + datafusion.io + + <ul> + <li><a href="autoapi/datafusion/io/index.html#module-datafusion.io">module</a> +</li> + </ul></li> + <li> datafusion.object_store <ul> @@ -1116,10 +1162,10 @@ </ul></li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_bin">date_bin() (in module datafusion.functions)</a> </li> - <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_part">date_part() (in module datafusion.functions)</a> -</li> </ul></td> <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_part">date_part() (in module datafusion.functions)</a> +</li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_trunc">date_trunc() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.datepart">datepart() (in module datafusion.functions)</a> @@ -1376,10 +1422,10 @@ <li><a href="autoapi/datafusion/index.html#datafusion.WindowFrame.get_lower_bound">(datafusion.WindowFrame method)</a> </li> </ul></li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.get_offset">get_offset() (datafusion.expr.WindowFrameBound method)</a> </li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowEvaluator.get_range">get_range() (datafusion.udf.WindowEvaluator method)</a> </li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrame.get_upper_bound">get_upper_bound() (datafusion.expr.WindowFrame method)</a> @@ -1388,6 +1434,8 @@ <li><a href="autoapi/datafusion/index.html#datafusion.WindowFrame.get_upper_bound">(datafusion.WindowFrame method)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.global_ctx">global_ctx() (datafusion.context.SessionContext class method)</a> +</li> <li><a href="autoapi/datafusion/object_store/index.html#datafusion.object_store.GoogleCloud">GoogleCloud (in module datafusion.object_store)</a> </li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.GroupingSet">GroupingSet (in module datafusion.expr)</a> @@ -1434,6 +1482,8 @@ </li> <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.intersect">intersect() (datafusion.dataframe.DataFrame method)</a> </li> + <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.into_view">into_view() (datafusion.dataframe.DataFrame method)</a> +</li> <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowEvaluator.is_causal">is_causal() (datafusion.udf.WindowEvaluator method)</a> </li> <li><a href="autoapi/datafusion/input/base/index.html#datafusion.input.base.BaseInputSource.is_correct_input">is_correct_input() (datafusion.input.base.BaseInputSource method)</a> @@ -1441,13 +1491,15 @@ <ul> <li><a href="autoapi/datafusion/input/location/index.html#datafusion.input.location.LocationInputPlugin.is_correct_input">(datafusion.input.location.LocationInputPlugin method)</a> </li> + <li><a href="autoapi/datafusion/input/index.html#datafusion.input.LocationInputPlugin.is_correct_input">(datafusion.input.LocationInputPlugin method)</a> +</li> </ul></li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.is_current_row">is_current_row() (datafusion.expr.WindowFrameBound method)</a> </li> - <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.is_following">is_following() (datafusion.expr.WindowFrameBound method)</a> -</li> </ul></td> <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.is_following">is_following() (datafusion.expr.WindowFrameBound method)</a> +</li> <li><a href="autoapi/datafusion/index.html#datafusion.Expr.is_not_null">is_not_null() (datafusion.Expr method)</a> <ul> @@ -1578,10 +1630,10 @@ </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_remove">list_remove() (in module datafusion.functions)</a> </li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_remove_all">list_remove_all() (in module datafusion.functions)</a> </li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_remove_n">list_remove_n() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_repeat">list_repeat() (in module datafusion.functions)</a> @@ -1618,8 +1670,12 @@ </li> <li><a href="autoapi/datafusion/object_store/index.html#datafusion.object_store.LocalFileSystem">LocalFileSystem (in module datafusion.object_store)</a> </li> - <li><a href="autoapi/datafusion/input/location/index.html#datafusion.input.location.LocationInputPlugin">LocationInputPlugin (class in datafusion.input.location)</a> + <li><a href="autoapi/datafusion/input/index.html#datafusion.input.LocationInputPlugin">LocationInputPlugin (class in datafusion.input)</a> + + <ul> + <li><a href="autoapi/datafusion/input/location/index.html#datafusion.input.location.LocationInputPlugin">(class in datafusion.input.location)</a> </li> + </ul></li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.log">log() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.log10">log10() (in module datafusion.functions)</a> @@ -1698,6 +1754,8 @@ </li> <li><a href="autoapi/datafusion/input/location/index.html#module-datafusion.input.location">datafusion.input.location</a> </li> + <li><a href="autoapi/datafusion/io/index.html#module-datafusion.io">datafusion.io</a> +</li> <li><a href="autoapi/datafusion/object_store/index.html#module-datafusion.object_store">datafusion.object_store</a> </li> <li><a href="autoapi/datafusion/plan/index.html#module-datafusion.plan">datafusion.plan</a> @@ -1864,13 +1922,37 @@ </li> </ul></li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_avro">read_avro() (datafusion.context.SessionContext method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.read_avro">(in module datafusion)</a> </li> + <li><a href="autoapi/datafusion/io/index.html#datafusion.io.read_avro">(in module datafusion.io)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_csv">read_csv() (datafusion.context.SessionContext method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.read_csv">(in module datafusion)</a> </li> + <li><a href="autoapi/datafusion/io/index.html#datafusion.io.read_csv">(in module datafusion.io)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_json">read_json() (datafusion.context.SessionContext method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.read_json">(in module datafusion)</a> </li> + <li><a href="autoapi/datafusion/io/index.html#datafusion.io.read_json">(in module datafusion.io)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_parquet">read_parquet() (datafusion.context.SessionContext method)</a> + + <ul> + <li><a href="autoapi/datafusion/index.html#datafusion.read_parquet">(in module datafusion)</a> </li> + <li><a href="autoapi/datafusion/io/index.html#datafusion.io.read_parquet">(in module datafusion.io)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_table">read_table() (datafusion.context.SessionContext method)</a> </li> <li><a href="autoapi/datafusion/record_batch/index.html#datafusion.record_batch.RecordBatch.record_batch">record_batch (datafusion.record_batch.RecordBatch attribute)</a> @@ -1891,6 +1973,8 @@ <li><a href="autoapi/datafusion/record_batch/index.html#datafusion.record_batch.RecordBatchStream">(class in datafusion.record_batch)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.regexp_count">regexp_count() (in module datafusion.functions)</a> +</li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.regexp_like">regexp_like() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.regexp_match">regexp_match() (in module datafusion.functions)</a> @@ -1909,6 +1993,8 @@ </li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_object_store">register_object_store() (datafusion.context.SessionContext method)</a> </li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_parquet">register_parquet() (datafusion.context.SessionContext method)</a> </li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_record_batches">register_record_batches() (datafusion.context.SessionContext method)</a> @@ -1917,14 +2003,14 @@ </li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_table_provider">register_table_provider() (datafusion.context.SessionContext method)</a> </li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_udaf">register_udaf() (datafusion.context.SessionContext method)</a> </li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_udf">register_udf() (datafusion.context.SessionContext method)</a> </li> <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_udwf">register_udwf() (datafusion.context.SessionContext method)</a> </li> + <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_view">register_view() (datafusion.context.SessionContext method)</a> +</li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.regr_avgx">regr_avgx() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.regr_avgy">regr_avgy() (in module datafusion.functions)</a> @@ -2007,14 +2093,14 @@ </ul></li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.ScalarVariable">ScalarVariable (in module datafusion.expr)</a> </li> - <li><a href="autoapi/datafusion/catalog/index.html#datafusion.catalog.Table.schema">schema() (datafusion.catalog.Table method)</a> + <li><a href="autoapi/datafusion/catalog/index.html#datafusion.catalog.Table.schema">schema (datafusion.catalog.Table property)</a> <ul> - <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.schema">(datafusion.dataframe.DataFrame method)</a> -</li> - <li><a href="autoapi/datafusion/index.html#datafusion.Table.schema">(datafusion.Table method)</a> + <li><a href="autoapi/datafusion/index.html#datafusion.Table.schema">(datafusion.Table property)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.schema">schema() (datafusion.dataframe.DataFrame method)</a> +</li> <li><a href="autoapi/datafusion/index.html#datafusion.Expr.schema_name">schema_name() (datafusion.Expr method)</a> <ul> @@ -2215,6 +2301,8 @@ </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_timestamp_millis">to_timestamp_millis() (in module datafusion.functions)</a> </li> + <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_timestamp_nanos">to_timestamp_nanos() (in module datafusion.functions)</a> +</li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_timestamp_seconds">to_timestamp_seconds() (in module datafusion.functions)</a> </li> <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_unixtime">to_unixtime() (in module datafusion.functions)</a> @@ -2251,32 +2339,50 @@ <h2 id="U">U</h2> <table style="width: 100%" class="indextable genindextable"><tr> <td style="width: 33%; vertical-align: top;"><ul> + <li><a href="autoapi/datafusion/index.html#datafusion.udaf">udaf (in module datafusion)</a> + + <ul> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.udaf">(in module datafusion.udf)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/index.html#datafusion.AggregateUDF.udaf">udaf() (datafusion.AggregateUDF static method)</a> <ul> <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.AggregateUDF.udaf">(datafusion.udf.AggregateUDF static method)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/index.html#datafusion.udf">udf (in module datafusion)</a> + + <ul> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.udf">(in module datafusion.udf)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/index.html#datafusion.ScalarUDF.udf">udf() (datafusion.ScalarUDF static method)</a> <ul> <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.ScalarUDF.udf">(datafusion.udf.ScalarUDF static method)</a> </li> </ul></li> + <li><a href="autoapi/datafusion/index.html#datafusion.udwf">udwf (in module datafusion)</a> + + <ul> + <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.udwf">(in module datafusion.udf)</a> +</li> + </ul></li> <li><a href="autoapi/datafusion/udf/index.html#datafusion.udf.WindowUDF.udwf">udwf() (datafusion.udf.WindowUDF static method)</a> <ul> <li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF.udwf">(datafusion.WindowUDF static method)</a> </li> </ul></li> + </ul></td> + <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.Compression.UNCOMPRESSED">UNCOMPRESSED (datafusion.dataframe.Compression attribute)</a> </li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Union">Union (in module datafusion.expr)</a> </li> <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.union">union() (datafusion.dataframe.DataFrame method)</a> </li> - </ul></td> - <td style="width: 33%; vertical-align: top;"><ul> <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.union_distinct">union_distinct() (datafusion.dataframe.DataFrame method)</a> </li> <li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Unnest">Unnest (in module datafusion.expr)</a>
diff --git a/index.html b/index.html index 5789052..2753909 100644 --- a/index.html +++ b/index.html
@@ -127,6 +127,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -232,6 +237,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -307,6 +317,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/objects.inv b/objects.inv index 2d6266b..b945840 100644 --- a/objects.inv +++ b/objects.inv Binary files differ
diff --git a/py-modindex.html b/py-modindex.html index 23fde1c..05bc2ec 100644 --- a/py-modindex.html +++ b/py-modindex.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -422,6 +437,11 @@ <tr class="cg-1"> <td></td> <td>    + <a href="autoapi/datafusion/io/index.html#module-datafusion.io"><code class="xref">datafusion.io</code></a></td><td> + <em></em></td></tr> + <tr class="cg-1"> + <td></td> + <td>    <a href="autoapi/datafusion/object_store/index.html#module-datafusion.object_store"><code class="xref">datafusion.object_store</code></a></td><td> <em></em></td></tr> <tr class="cg-1">
diff --git a/search.html b/search.html index acc066c..6461c91 100644 --- a/search.html +++ b/search.html
@@ -132,6 +132,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="user-guide/common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="user-guide/common-operations/basic-info.html"> Basic Operations </a> @@ -237,6 +242,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -312,6 +322,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/searchindex.js b/searchindex.js index b908d33..143eae6 100644 --- a/searchindex.js +++ b/searchindex.js
@@ -1 +1 @@ -Search.setIndex({"alltitles": {"API Reference": [[14, null]], "Aggregate Functions": [[18, "aggregate-functions"], [25, "aggregate-functions"], [26, "aggregate-functions"]], "Aggregation": [[18, null]], "Arrays": [[20, "arrays"]], "Arrow": [[30, null]], "Attributes": [[3, "attributes"], [5, "attributes"], [9, "attributes"], [13, "attributes"]], "Available Functions": [[26, "available-functions"]], "Avro": [[31, null]], "Basic Operations": [[19, null]], "Boolean": [[20, "boolean"]], "CSV": [[32, null]], "Casting": [[21, "casting"]], "Classes": [[0, "classes"], [1, "classes"], [2, "classes"], [3, "classes"], [5, "classes"], [6, "classes"], [8, "classes"], [10, "classes"], [11, "classes"], [12, "classes"], [13, "classes"]], "Column": [[20, "column"]], "Column Selections": [[24, null]], "Common Operations": [[22, null]], "Concepts": [[17, null]], "Conditional": [[21, "conditional"]], "Configuration": [[27, null]], "Create in-memory": [[28, "create-in-memory"]], "Custom Table Provider": [[28, "custom-table-provider"], [36, null]], "Data Sources": [[28, null]], "DataFrame": [[17, "dataframe"]], "DataFusion in Python": [[16, null]], "Delta Lake": [[28, "delta-lake"]], "Distinct": [[18, "distinct"]], "Example": [[16, "example"]], "Exporting from DataFusion": [[30, "exporting-from-datafusion"]], "Expressions": [[17, "expressions"], [20, null]], "Filter": [[18, "filter"]], "Full Join": [[23, "full-join"]], "Functions": [[4, "functions"], [5, "functions"], [20, "functions"], [21, null]], "Guidelines for Separating Python and Rust Code": [[15, "guidelines-for-separating-python-and-rust-code"]], "How to develop": [[15, "how-to-develop"]], "IO": [[33, null]], "Iceberg": [[28, "iceberg"]], "Importing to DataFusion": [[30, "importing-to-datafusion"]], "Improving Build Speed": [[15, "improving-build-speed"]], "Inner Join": [[23, "inner-join"]], "Install": [[16, "install"]], "Installation": [[29, "installation"]], "Introduction": [[15, null], [29, null]], "JSON": [[34, null]], "Joins": [[23, null]], "Left Anti Join": [[23, "left-anti-join"]], "Left Join": [[23, "left-join"]], "Left Semi Join": [[23, "left-semi-join"]], "Literal": [[20, "literal"]], "Local file": [[28, "local-file"]], "Mathematical": [[21, "mathematical"]], "Module Contents": [[0, "module-contents"], [1, "module-contents"], [2, "module-contents"], [3, "module-contents"], [4, "module-contents"], [6, "module-contents"], [8, "module-contents"], [9, "module-contents"], [10, "module-contents"], [11, "module-contents"], [12, "module-contents"], [13, "module-contents"]], "Null Treatment": [[18, "null-treatment"], [26, "null-treatment"]], "Object Store": [[28, "object-store"]], "Ordering": [[18, "ordering"], [26, "ordering"]], "Other": [[21, "other"]], "Other DataFrame Libraries": [[28, "other-dataframe-libraries"]], "Package Contents": [[5, "package-contents"]], "Parquet": [[35, null]], "Partitions": [[26, "partitions"]], "Returns:": [[4, "returns"], [4, "id1"]], "Running & Installing pre-commit hooks": [[15, "running-installing-pre-commit-hooks"]], "SQL": [[37, null]], "Scalar Functions": [[25, "scalar-functions"]], "Session Context": [[17, "session-context"]], "Setting Parameters": [[18, "setting-parameters"], [26, "setting-parameters"]], "String": [[21, "string"]], "Structs": [[20, "structs"]], "Submodules": [[5, "submodules"], [7, "submodules"]], "Temporal": [[21, "temporal"]], "UDWF options": [[25, "udwf-options"]], "Update Dependencies": [[15, "update-dependencies"]], "User-Defined Functions": [[25, null]], "Window Frame": [[26, "window-frame"]], "Window Functions": [[25, "window-functions"], [26, null]], "datafusion": [[5, null]], "datafusion.catalog": [[0, null]], "datafusion.context": [[1, null]], "datafusion.dataframe": [[2, null]], "datafusion.expr": [[3, null]], "datafusion.functions": [[4, null]], "datafusion.input": [[7, null]], "datafusion.input.base": [[6, null]], "datafusion.input.location": [[8, null]], "datafusion.object_store": [[9, null]], "datafusion.plan": [[10, null]], "datafusion.record_batch": [[11, null]], "datafusion.substrait": [[12, null]], "datafusion.udf": [[13, null]]}, "docnames": ["autoapi/datafusion/catalog/index", "autoapi/datafusion/context/index", "autoapi/datafusion/dataframe/index", "autoapi/datafusion/expr/index", "autoapi/datafusion/functions/index", "autoapi/datafusion/index", "autoapi/datafusion/input/base/index", "autoapi/datafusion/input/index", "autoapi/datafusion/input/location/index", "autoapi/datafusion/object_store/index", "autoapi/datafusion/plan/index", "autoapi/datafusion/record_batch/index", "autoapi/datafusion/substrait/index", "autoapi/datafusion/udf/index", "autoapi/index", "contributor-guide/introduction", "index", "user-guide/basics", "user-guide/common-operations/aggregations", "user-guide/common-operations/basic-info", "user-guide/common-operations/expressions", "user-guide/common-operations/functions", "user-guide/common-operations/index", "user-guide/common-operations/joins", "user-guide/common-operations/select-and-filter", "user-guide/common-operations/udf-and-udfa", "user-guide/common-operations/windows", "user-guide/configuration", "user-guide/data-sources", "user-guide/introduction", "user-guide/io/arrow", "user-guide/io/avro", "user-guide/io/csv", "user-guide/io/index", "user-guide/io/json", "user-guide/io/parquet", "user-guide/io/table_provider", "user-guide/sql"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["autoapi/datafusion/catalog/index.rst", "autoapi/datafusion/context/index.rst", "autoapi/datafusion/dataframe/index.rst", "autoapi/datafusion/expr/index.rst", "autoapi/datafusion/functions/index.rst", "autoapi/datafusion/index.rst", "autoapi/datafusion/input/base/index.rst", "autoapi/datafusion/input/index.rst", "autoapi/datafusion/input/location/index.rst", "autoapi/datafusion/object_store/index.rst", "autoapi/datafusion/plan/index.rst", "autoapi/datafusion/record_batch/index.rst", "autoapi/datafusion/substrait/index.rst", "autoapi/datafusion/udf/index.rst", "autoapi/index.rst", "contributor-guide/introduction.rst", "index.rst", "user-guide/basics.rst", "user-guide/common-operations/aggregations.rst", "user-guide/common-operations/basic-info.rst", "user-guide/common-operations/expressions.rst", "user-guide/common-operations/functions.rst", "user-guide/common-operations/index.rst", "user-guide/common-operations/joins.rst", "user-guide/common-operations/select-and-filter.rst", "user-guide/common-operations/udf-and-udfa.rst", "user-guide/common-operations/windows.rst", "user-guide/configuration.rst", "user-guide/data-sources.rst", "user-guide/introduction.rst", "user-guide/io/arrow.rst", "user-guide/io/avro.rst", "user-guide/io/csv.rst", "user-guide/io/index.rst", "user-guide/io/json.rst", "user-guide/io/parquet.rst", "user-guide/io/table_provider.rst", "user-guide/sql.rst"], "indexentries": {"__add__() (datafusion.expr method)": [[5, "datafusion.Expr.__add__", false]], "__add__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__add__", false]], "__aiter__() (datafusion.record_batch.recordbatchstream method)": [[11, "datafusion.record_batch.RecordBatchStream.__aiter__", false]], "__aiter__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__aiter__", false]], "__and__() (datafusion.expr method)": [[5, "datafusion.Expr.__and__", false]], "__and__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__and__", false]], "__anext__() (datafusion.record_batch.recordbatchstream method)": [[11, "datafusion.record_batch.RecordBatchStream.__anext__", false]], "__anext__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__anext__", false]], "__arrow_c_array__() (datafusion.context.arrowarrayexportable method)": [[1, "datafusion.context.ArrowArrayExportable.__arrow_c_array__", false]], "__arrow_c_stream__() (datafusion.context.arrowstreamexportable method)": [[1, "datafusion.context.ArrowStreamExportable.__arrow_c_stream__", false]], "__arrow_c_stream__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__arrow_c_stream__", false]], "__call__() (datafusion.aggregateudf method)": [[5, "datafusion.AggregateUDF.__call__", false]], "__call__() (datafusion.scalarudf method)": [[5, "datafusion.ScalarUDF.__call__", false]], "__call__() (datafusion.udf.aggregateudf method)": [[13, "datafusion.udf.AggregateUDF.__call__", false]], "__call__() (datafusion.udf.scalarudf method)": [[13, "datafusion.udf.ScalarUDF.__call__", false]], "__call__() (datafusion.udf.windowudf method)": [[13, "datafusion.udf.WindowUDF.__call__", false]], "__call__() (datafusion.windowudf method)": [[5, "datafusion.WindowUDF.__call__", false]], "__datafusion_table_provider__() (datafusion.context.tableproviderexportable method)": [[1, "datafusion.context.TableProviderExportable.__datafusion_table_provider__", false]], "__eq__() (datafusion.expr method)": [[5, "datafusion.Expr.__eq__", false]], "__eq__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__eq__", false]], "__ge__() (datafusion.expr method)": [[5, "datafusion.Expr.__ge__", false]], "__ge__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__ge__", false]], "__getitem__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__getitem__", false]], "__getitem__() (datafusion.expr method)": [[5, "datafusion.Expr.__getitem__", false]], "__getitem__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__getitem__", false]], "__gt__() (datafusion.expr method)": [[5, "datafusion.Expr.__gt__", false]], "__gt__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__gt__", false]], "__invert__() (datafusion.expr method)": [[5, "datafusion.Expr.__invert__", false]], "__invert__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__invert__", false]], "__iter__() (datafusion.record_batch.recordbatchstream method)": [[11, "datafusion.record_batch.RecordBatchStream.__iter__", false]], "__iter__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__iter__", false]], "__le__() (datafusion.expr method)": [[5, "datafusion.Expr.__le__", false]], "__le__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__le__", false]], "__lt__() (datafusion.expr method)": [[5, "datafusion.Expr.__lt__", false]], "__lt__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__lt__", false]], "__mod__() (datafusion.expr method)": [[5, "datafusion.Expr.__mod__", false]], "__mod__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__mod__", false]], "__mul__() (datafusion.expr method)": [[5, "datafusion.Expr.__mul__", false]], "__mul__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__mul__", false]], "__ne__() (datafusion.expr method)": [[5, "datafusion.Expr.__ne__", false]], "__ne__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__ne__", false]], "__next__() (datafusion.record_batch.recordbatchstream method)": [[11, "datafusion.record_batch.RecordBatchStream.__next__", false]], "__next__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__next__", false]], "__or__() (datafusion.expr method)": [[5, "datafusion.Expr.__or__", false]], "__or__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__or__", false]], "__radd__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__radd__", false]], "__radd__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__radd__", false]], "__rand__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rand__", false]], "__rand__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rand__", false]], "__repr__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__repr__", false]], "__repr__() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.__repr__", false]], "__repr__() (datafusion.expr method)": [[5, "datafusion.Expr.__repr__", false]], "__repr__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__repr__", false]], "__repr__() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.__repr__", false]], "__repr__() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.__repr__", false]], "__repr__() (datafusion.plan.executionplan method)": [[10, "datafusion.plan.ExecutionPlan.__repr__", false]], "__repr__() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.__repr__", false]], "__richcmp__() (datafusion.expr method)": [[5, "datafusion.Expr.__richcmp__", false]], "__richcmp__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__richcmp__", false]], "__rmod__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rmod__", false]], "__rmod__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rmod__", false]], "__rmul__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rmul__", false]], "__rmul__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rmul__", false]], "__ror__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__ror__", false]], "__ror__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__ror__", false]], "__rsub__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rsub__", false]], "__rsub__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rsub__", false]], "__rtruediv__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rtruediv__", false]], "__rtruediv__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rtruediv__", false]], "__str__() (datafusion.udf.volatility method)": [[13, "datafusion.udf.Volatility.__str__", false]], "__sub__() (datafusion.expr method)": [[5, "datafusion.Expr.__sub__", false]], "__sub__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__sub__", false]], "__truediv__() (datafusion.expr method)": [[5, "datafusion.Expr.__truediv__", false]], "__truediv__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__truediv__", false]], "_null_treatment (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._null_treatment", false]], "_order_by (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._order_by", false]], "_partition_by (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._partition_by", false]], "_r (in module datafusion.udf)": [[13, "datafusion.udf._R", false]], "_raw_plan (datafusion.executionplan attribute)": [[5, "datafusion.ExecutionPlan._raw_plan", false]], "_raw_plan (datafusion.logicalplan attribute)": [[5, "datafusion.LogicalPlan._raw_plan", false]], "_raw_plan (datafusion.plan.executionplan attribute)": [[10, "datafusion.plan.ExecutionPlan._raw_plan", false]], "_raw_plan (datafusion.plan.logicalplan attribute)": [[10, "datafusion.plan.LogicalPlan._raw_plan", false]], "_repr_html_() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame._repr_html_", false]], "_to_pyarrow_types (datafusion.expr attribute)": [[5, "datafusion.Expr._to_pyarrow_types", false]], "_to_pyarrow_types (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr._to_pyarrow_types", false]], "_udaf (datafusion.aggregateudf attribute)": [[5, "datafusion.AggregateUDF._udaf", false]], "_udaf (datafusion.udf.aggregateudf attribute)": [[13, "datafusion.udf.AggregateUDF._udaf", false]], "_udf (datafusion.scalarudf attribute)": [[5, "datafusion.ScalarUDF._udf", false]], "_udf (datafusion.udf.scalarudf attribute)": [[13, "datafusion.udf.ScalarUDF._udf", false]], "_udwf (datafusion.udf.windowudf attribute)": [[13, "datafusion.udf.WindowUDF._udwf", false]], "_udwf (datafusion.windowudf attribute)": [[5, "datafusion.WindowUDF._udwf", false]], "_w (in module datafusion.udf)": [[13, "datafusion.udf._W", false]], "_window_frame (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._window_frame", false]], "abs() (in module datafusion.functions)": [[4, "datafusion.functions.abs", false]], "accumulator (class in datafusion)": [[5, "datafusion.Accumulator", false]], "accumulator (class in datafusion.udf)": [[13, "datafusion.udf.Accumulator", false]], "acos() (in module datafusion.functions)": [[4, "datafusion.functions.acos", false]], "acosh() (in module datafusion.functions)": [[4, "datafusion.functions.acosh", false]], "aggregate (in module datafusion.expr)": [[3, "datafusion.expr.Aggregate", false]], "aggregate() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.aggregate", false]], "aggregatefunction (in module datafusion.expr)": [[3, "datafusion.expr.AggregateFunction", false]], "aggregateudf (class in datafusion)": [[5, "datafusion.AggregateUDF", false]], "aggregateudf (class in datafusion.udf)": [[13, "datafusion.udf.AggregateUDF", false]], "alias (in module datafusion.expr)": [[3, "datafusion.expr.Alias", false]], "alias() (datafusion.expr method)": [[5, "datafusion.Expr.alias", false]], "alias() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.alias", false]], "alias() (in module datafusion.functions)": [[4, "datafusion.functions.alias", false]], "amazons3 (in module datafusion.object_store)": [[9, "datafusion.object_store.AmazonS3", false]], "analyze (in module datafusion.expr)": [[3, "datafusion.expr.Analyze", false]], "approx_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.approx_distinct", false]], "approx_median() (in module datafusion.functions)": [[4, "datafusion.functions.approx_median", false]], "approx_percentile_cont() (in module datafusion.functions)": [[4, "datafusion.functions.approx_percentile_cont", false]], "approx_percentile_cont_with_weight() (in module datafusion.functions)": [[4, "datafusion.functions.approx_percentile_cont_with_weight", false]], "array() (in module datafusion.functions)": [[4, "datafusion.functions.array", false]], "array_agg() (in module datafusion.functions)": [[4, "datafusion.functions.array_agg", false]], "array_append() (in module datafusion.functions)": [[4, "datafusion.functions.array_append", false]], "array_cat() (in module datafusion.functions)": [[4, "datafusion.functions.array_cat", false]], "array_concat() (in module datafusion.functions)": [[4, "datafusion.functions.array_concat", false]], "array_dims() (in module datafusion.functions)": [[4, "datafusion.functions.array_dims", false]], "array_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.array_distinct", false]], "array_element() (in module datafusion.functions)": [[4, "datafusion.functions.array_element", false]], "array_empty() (in module datafusion.functions)": [[4, "datafusion.functions.array_empty", false]], "array_except() (in module datafusion.functions)": [[4, "datafusion.functions.array_except", false]], "array_extract() (in module datafusion.functions)": [[4, "datafusion.functions.array_extract", false]], "array_has() (in module datafusion.functions)": [[4, "datafusion.functions.array_has", false]], "array_has_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_has_all", false]], "array_has_any() (in module datafusion.functions)": [[4, "datafusion.functions.array_has_any", false]], "array_indexof() (in module datafusion.functions)": [[4, "datafusion.functions.array_indexof", false]], "array_intersect() (in module datafusion.functions)": [[4, "datafusion.functions.array_intersect", false]], "array_join() (in module datafusion.functions)": [[4, "datafusion.functions.array_join", false]], "array_length() (in module datafusion.functions)": [[4, "datafusion.functions.array_length", false]], "array_ndims() (in module datafusion.functions)": [[4, "datafusion.functions.array_ndims", false]], "array_pop_back() (in module datafusion.functions)": [[4, "datafusion.functions.array_pop_back", false]], "array_pop_front() (in module datafusion.functions)": [[4, "datafusion.functions.array_pop_front", false]], "array_position() (in module datafusion.functions)": [[4, "datafusion.functions.array_position", false]], "array_positions() (in module datafusion.functions)": [[4, "datafusion.functions.array_positions", false]], "array_prepend() (in module datafusion.functions)": [[4, "datafusion.functions.array_prepend", false]], "array_push_back() (in module datafusion.functions)": [[4, "datafusion.functions.array_push_back", false]], "array_push_front() (in module datafusion.functions)": [[4, "datafusion.functions.array_push_front", false]], "array_remove() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove", false]], "array_remove_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove_all", false]], "array_remove_n() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove_n", false]], "array_repeat() (in module datafusion.functions)": [[4, "datafusion.functions.array_repeat", false]], "array_replace() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace", false]], "array_replace_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace_all", false]], "array_replace_n() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace_n", false]], "array_resize() (in module datafusion.functions)": [[4, "datafusion.functions.array_resize", false]], "array_slice() (in module datafusion.functions)": [[4, "datafusion.functions.array_slice", false]], "array_sort() (in module datafusion.functions)": [[4, "datafusion.functions.array_sort", false]], "array_to_string() (in module datafusion.functions)": [[4, "datafusion.functions.array_to_string", false]], "array_union() (in module datafusion.functions)": [[4, "datafusion.functions.array_union", false]], "arrow_cast() (in module datafusion.functions)": [[4, "datafusion.functions.arrow_cast", false]], "arrow_typeof() (in module datafusion.functions)": [[4, "datafusion.functions.arrow_typeof", false]], "arrowarrayexportable (class in datafusion.context)": [[1, "datafusion.context.ArrowArrayExportable", false]], "arrowstreamexportable (class in datafusion.context)": [[1, "datafusion.context.ArrowStreamExportable", false]], "ascending() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.ascending", false]], "ascii() (in module datafusion.functions)": [[4, "datafusion.functions.ascii", false]], "asin() (in module datafusion.functions)": [[4, "datafusion.functions.asin", false]], "asinh() (in module datafusion.functions)": [[4, "datafusion.functions.asinh", false]], "atan() (in module datafusion.functions)": [[4, "datafusion.functions.atan", false]], "atan2() (in module datafusion.functions)": [[4, "datafusion.functions.atan2", false]], "atanh() (in module datafusion.functions)": [[4, "datafusion.functions.atanh", false]], "avg() (in module datafusion.functions)": [[4, "datafusion.functions.avg", false]], "baseinputsource (class in datafusion.input.base)": [[6, "datafusion.input.base.BaseInputSource", false]], "between (in module datafusion.expr)": [[3, "datafusion.expr.Between", false]], "between() (datafusion.expr method)": [[5, "datafusion.Expr.between", false]], "between() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.between", false]], "binaryexpr (in module datafusion.expr)": [[3, "datafusion.expr.BinaryExpr", false]], "bit_and() (in module datafusion.functions)": [[4, "datafusion.functions.bit_and", false]], "bit_length() (in module datafusion.functions)": [[4, "datafusion.functions.bit_length", false]], "bit_or() (in module datafusion.functions)": [[4, "datafusion.functions.bit_or", false]], "bit_xor() (in module datafusion.functions)": [[4, "datafusion.functions.bit_xor", false]], "bool_and() (in module datafusion.functions)": [[4, "datafusion.functions.bool_and", false]], "bool_or() (in module datafusion.functions)": [[4, "datafusion.functions.bool_or", false]], "brotli (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.BROTLI", false]], "btrim() (in module datafusion.functions)": [[4, "datafusion.functions.btrim", false]], "build_table() (datafusion.input.base.baseinputsource method)": [[6, "datafusion.input.base.BaseInputSource.build_table", false]], "build_table() (datafusion.input.location.locationinputplugin method)": [[8, "datafusion.input.location.LocationInputPlugin.build_table", false]], "cache() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.cache", false]], "canonical_name() (datafusion.expr method)": [[5, "datafusion.Expr.canonical_name", false]], "canonical_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.canonical_name", false]], "cardinality() (in module datafusion.functions)": [[4, "datafusion.functions.cardinality", false]], "case (in module datafusion.expr)": [[3, "datafusion.expr.Case", false]], "case() (in module datafusion.functions)": [[4, "datafusion.functions.case", false]], "case_builder (datafusion.expr.casebuilder attribute)": [[3, "datafusion.expr.CaseBuilder.case_builder", false]], "casebuilder (class in datafusion.expr)": [[3, "datafusion.expr.CaseBuilder", false]], "cast (in module datafusion.expr)": [[3, "datafusion.expr.Cast", false]], "cast() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.cast", false]], "cast() (datafusion.expr method)": [[5, "datafusion.Expr.cast", false]], "cast() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.cast", false]], "catalog (class in datafusion)": [[5, "datafusion.Catalog", false]], "catalog (class in datafusion.catalog)": [[0, "datafusion.catalog.Catalog", false]], "catalog (datafusion.catalog attribute)": [[5, "datafusion.Catalog.catalog", false]], "catalog (datafusion.catalog.catalog attribute)": [[0, "datafusion.catalog.Catalog.catalog", false]], "catalog() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.catalog", false]], "cbrt() (in module datafusion.functions)": [[4, "datafusion.functions.cbrt", false]], "ceil() (in module datafusion.functions)": [[4, "datafusion.functions.ceil", false]], "char_length() (in module datafusion.functions)": [[4, "datafusion.functions.char_length", false]], "character_length() (in module datafusion.functions)": [[4, "datafusion.functions.character_length", false]], "children() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.children", false]], "children() (datafusion.plan.executionplan method)": [[10, "datafusion.plan.ExecutionPlan.children", false]], "chr() (in module datafusion.functions)": [[4, "datafusion.functions.chr", false]], "coalesce() (in module datafusion.functions)": [[4, "datafusion.functions.coalesce", false]], "col() (in module datafusion)": [[5, "datafusion.col", false]], "col() (in module datafusion.functions)": [[4, "datafusion.functions.col", false]], "collect() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.collect", false]], "collect_partitioned() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.collect_partitioned", false]], "column (in module datafusion.expr)": [[3, "datafusion.expr.Column", false]], "column() (datafusion.expr static method)": [[5, "datafusion.Expr.column", false]], "column() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.column", false]], "column() (in module datafusion)": [[5, "datafusion.column", false]], "column_name() (datafusion.expr method)": [[5, "datafusion.Expr.column_name", false]], "column_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.column_name", false]], "compression (class in datafusion.dataframe)": [[2, "datafusion.dataframe.Compression", false]], "concat() (in module datafusion.functions)": [[4, "datafusion.functions.concat", false]], "concat_ws() (in module datafusion.functions)": [[4, "datafusion.functions.concat_ws", false]], "config_internal (datafusion.context.runtimeenvbuilder attribute)": [[1, "datafusion.context.RuntimeEnvBuilder.config_internal", false]], "config_internal (datafusion.context.sessionconfig attribute)": [[1, "datafusion.context.SessionConfig.config_internal", false]], "config_internal (datafusion.runtimeenvbuilder attribute)": [[5, "datafusion.RuntimeEnvBuilder.config_internal", false]], "config_internal (datafusion.sessionconfig attribute)": [[5, "datafusion.SessionConfig.config_internal", false]], "consumer (class in datafusion.substrait)": [[12, "datafusion.substrait.Consumer", false]], "corr() (in module datafusion.functions)": [[4, "datafusion.functions.corr", false]], "cos() (in module datafusion.functions)": [[4, "datafusion.functions.cos", false]], "cosh() (in module datafusion.functions)": [[4, "datafusion.functions.cosh", false]], "cot() (in module datafusion.functions)": [[4, "datafusion.functions.cot", false]], "count() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.count", false]], "count() (in module datafusion.functions)": [[4, "datafusion.functions.count", false]], "count_star() (in module datafusion.functions)": [[4, "datafusion.functions.count_star", false]], "covar() (in module datafusion.functions)": [[4, "datafusion.functions.covar", false]], "covar_pop() (in module datafusion.functions)": [[4, "datafusion.functions.covar_pop", false]], "covar_samp() (in module datafusion.functions)": [[4, "datafusion.functions.covar_samp", false]], "create_dataframe() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.create_dataframe", false]], "create_dataframe_from_logical_plan() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.create_dataframe_from_logical_plan", false]], "creatememorytable (in module datafusion.expr)": [[3, "datafusion.expr.CreateMemoryTable", false]], "createview (in module datafusion.expr)": [[3, "datafusion.expr.CreateView", false]], "ctx (datafusion.context.sessioncontext attribute)": [[1, "datafusion.context.SessionContext.ctx", false]], "cume_dist() (in module datafusion.functions)": [[4, "datafusion.functions.cume_dist", false]], "current_date() (in module datafusion.functions)": [[4, "datafusion.functions.current_date", false]], "current_time() (in module datafusion.functions)": [[4, "datafusion.functions.current_time", false]], "database (class in datafusion)": [[5, "datafusion.Database", false]], "database (class in datafusion.catalog)": [[0, "datafusion.catalog.Database", false]], "database() (datafusion.catalog method)": [[5, "datafusion.Catalog.database", false]], "database() (datafusion.catalog.catalog method)": [[0, "datafusion.catalog.Catalog.database", false]], "dataframe (class in datafusion.dataframe)": [[2, "datafusion.dataframe.DataFrame", false]], "datafusion": [[5, "module-datafusion", false]], "datafusion.catalog": [[0, "module-datafusion.catalog", false]], "datafusion.context": [[1, "module-datafusion.context", false]], "datafusion.dataframe": [[2, "module-datafusion.dataframe", false]], "datafusion.expr": [[3, "module-datafusion.expr", false]], "datafusion.functions": [[4, "module-datafusion.functions", false]], "datafusion.input": [[7, "module-datafusion.input", false]], "datafusion.input.base": [[6, "module-datafusion.input.base", false]], "datafusion.input.location": [[8, "module-datafusion.input.location", false]], "datafusion.object_store": [[9, "module-datafusion.object_store", false]], "datafusion.plan": [[10, "module-datafusion.plan", false]], "datafusion.record_batch": [[11, "module-datafusion.record_batch", false]], "datafusion.substrait": [[12, "module-datafusion.substrait", false]], "datafusion.udf": [[13, "module-datafusion.udf", false]], "date_bin() (in module datafusion.functions)": [[4, "datafusion.functions.date_bin", false]], "date_part() (in module datafusion.functions)": [[4, "datafusion.functions.date_part", false]], "date_trunc() (in module datafusion.functions)": [[4, "datafusion.functions.date_trunc", false]], "datepart() (in module datafusion.functions)": [[4, "datafusion.functions.datepart", false]], "datetrunc() (in module datafusion.functions)": [[4, "datafusion.functions.datetrunc", false]], "db (datafusion.catalog.database attribute)": [[0, "datafusion.catalog.Database.db", false]], "db (datafusion.database attribute)": [[5, "datafusion.Database.db", false]], "decode() (in module datafusion.functions)": [[4, "datafusion.functions.decode", false]], "degrees() (in module datafusion.functions)": [[4, "datafusion.functions.degrees", false]], "dense_rank() (in module datafusion.functions)": [[4, "datafusion.functions.dense_rank", false]], "deregister_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.deregister_table", false]], "describe() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.describe", false]], "deserialize() (datafusion.substrait.serde static method)": [[12, "datafusion.substrait.Serde.deserialize", false]], "deserialize_bytes() (datafusion.substrait.serde static method)": [[12, "datafusion.substrait.Serde.deserialize_bytes", false]], "df (datafusion.dataframe.dataframe attribute)": [[2, "datafusion.dataframe.DataFrame.df", false]], "dfschema (in module datafusion)": [[5, "datafusion.DFSchema", false]], "digest() (in module datafusion.functions)": [[4, "datafusion.functions.digest", false]], "display() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.display", false]], "display() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display", false]], "display() (datafusion.plan.executionplan method)": [[10, "datafusion.plan.ExecutionPlan.display", false]], "display() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.display", false]], "display_graphviz() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_graphviz", false]], "display_graphviz() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.display_graphviz", false]], "display_indent() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.display_indent", false]], "display_indent() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_indent", false]], "display_indent() (datafusion.plan.executionplan method)": [[10, "datafusion.plan.ExecutionPlan.display_indent", false]], "display_indent() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.display_indent", false]], "display_indent_schema() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_indent_schema", false]], "display_indent_schema() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.display_indent_schema", false]], "display_name() (datafusion.expr method)": [[5, "datafusion.Expr.display_name", false]], "display_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.display_name", false]], "distinct (in module datafusion.expr)": [[3, "datafusion.expr.Distinct", false]], "distinct() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.distinct", false]], "distinct() (datafusion.expr method)": [[5, "datafusion.Expr.distinct", false]], "distinct() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.distinct", false]], "drop() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.drop", false]], "droptable (in module datafusion.expr)": [[3, "datafusion.expr.DropTable", false]], "empty() (in module datafusion.functions)": [[4, "datafusion.functions.empty", false]], "empty_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.empty_table", false]], "emptyrelation (in module datafusion.expr)": [[3, "datafusion.expr.EmptyRelation", false]], "enable_url_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.enable_url_table", false]], "encode() (datafusion.substrait.plan method)": [[12, "datafusion.substrait.Plan.encode", false]], "encode() (in module datafusion.functions)": [[4, "datafusion.functions.encode", false]], "end() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.end", false]], "ends_with() (in module datafusion.functions)": [[4, "datafusion.functions.ends_with", false]], "evaluate() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.evaluate", false]], "evaluate() (datafusion.udf.accumulator method)": [[13, "datafusion.udf.Accumulator.evaluate", false]], "evaluate() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.evaluate", false]], "evaluate_all() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.evaluate_all", false]], "evaluate_all_with_rank() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.evaluate_all_with_rank", false]], "except_all() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.except_all", false]], "execute() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.execute", false]], "execute_stream() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execute_stream", false]], "execute_stream_partitioned() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execute_stream_partitioned", false]], "execution_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execution_plan", false]], "executionplan (class in datafusion)": [[5, "datafusion.ExecutionPlan", false]], "executionplan (class in datafusion.plan)": [[10, "datafusion.plan.ExecutionPlan", false]], "exists (in module datafusion.expr)": [[3, "datafusion.expr.Exists", false]], "exp() (in module datafusion.functions)": [[4, "datafusion.functions.exp", false]], "explain (in module datafusion.expr)": [[3, "datafusion.expr.Explain", false]], "explain() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.explain", false]], "expr (class in datafusion)": [[5, "datafusion.Expr", false]], "expr (class in datafusion.expr)": [[3, "datafusion.expr.Expr", false]], "expr (datafusion.expr attribute)": [[5, "datafusion.Expr.expr", false]], "expr (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.expr", false]], "expr() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.expr", false]], "extension (in module datafusion.expr)": [[3, "datafusion.expr.Extension", false]], "extract() (in module datafusion.functions)": [[4, "datafusion.functions.extract", false]], "factorial() (in module datafusion.functions)": [[4, "datafusion.functions.factorial", false]], "fill_nan() (datafusion.expr method)": [[5, "datafusion.Expr.fill_nan", false]], "fill_nan() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.fill_nan", false]], "fill_null() (datafusion.expr method)": [[5, "datafusion.Expr.fill_null", false]], "fill_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.fill_null", false]], "filter (in module datafusion.expr)": [[3, "datafusion.expr.Filter", false]], "filter() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.filter", false]], "filter() (datafusion.expr method)": [[5, "datafusion.Expr.filter", false]], "filter() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.filter", false]], "find_in_set() (in module datafusion.functions)": [[4, "datafusion.functions.find_in_set", false]], "first_value() (in module datafusion.functions)": [[4, "datafusion.functions.first_value", false]], "flatten() (in module datafusion.functions)": [[4, "datafusion.functions.flatten", false]], "floor() (in module datafusion.functions)": [[4, "datafusion.functions.floor", false]], "frame_bound (datafusion.expr.windowframebound attribute)": [[3, "datafusion.expr.WindowFrameBound.frame_bound", false]], "from_arrow() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_arrow", false]], "from_arrow_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_arrow_table", false]], "from_pandas() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pandas", false]], "from_polars() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_polars", false]], "from_proto() (datafusion.executionplan static method)": [[5, "datafusion.ExecutionPlan.from_proto", false]], "from_proto() (datafusion.logicalplan static method)": [[5, "datafusion.LogicalPlan.from_proto", false]], "from_proto() (datafusion.plan.executionplan static method)": [[10, "datafusion.plan.ExecutionPlan.from_proto", false]], "from_proto() (datafusion.plan.logicalplan static method)": [[10, "datafusion.plan.LogicalPlan.from_proto", false]], "from_pydict() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pydict", false]], "from_pylist() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pylist", false]], "from_str() (datafusion.dataframe.compression class method)": [[2, "datafusion.dataframe.Compression.from_str", false]], "from_substrait_plan() (datafusion.substrait.consumer static method)": [[12, "datafusion.substrait.Consumer.from_substrait_plan", false]], "from_unixtime() (in module datafusion.functions)": [[4, "datafusion.functions.from_unixtime", false]], "gcd() (in module datafusion.functions)": [[4, "datafusion.functions.gcd", false]], "get_default_level() (datafusion.dataframe.compression method)": [[2, "datafusion.dataframe.Compression.get_default_level", false]], "get_frame_units() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_frame_units", false]], "get_frame_units() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_frame_units", false]], "get_lower_bound() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_lower_bound", false]], "get_lower_bound() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_lower_bound", false]], "get_offset() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.get_offset", false]], "get_range() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.get_range", false]], "get_upper_bound() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_upper_bound", false]], "get_upper_bound() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_upper_bound", false]], "googlecloud (in module datafusion.object_store)": [[9, "datafusion.object_store.GoogleCloud", false]], "groupingset (in module datafusion.expr)": [[3, "datafusion.expr.GroupingSet", false]], "gzip (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.GZIP", false]], "head() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.head", false]], "http (in module datafusion.object_store)": [[9, "datafusion.object_store.Http", false]], "ilike (in module datafusion.expr)": [[3, "datafusion.expr.ILike", false]], "immutable (datafusion.udf.volatility attribute)": [[13, "datafusion.udf.Volatility.Immutable", false]], "in_list() (in module datafusion.functions)": [[4, "datafusion.functions.in_list", false]], "include_rank() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.include_rank", false]], "initcap() (in module datafusion.functions)": [[4, "datafusion.functions.initcap", false]], "inlist (in module datafusion.expr)": [[3, "datafusion.expr.InList", false]], "inputs() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.inputs", false]], "inputs() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.inputs", false]], "insubquery (in module datafusion.expr)": [[3, "datafusion.expr.InSubquery", false]], "intersect() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.intersect", false]], "is_causal() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.is_causal", false]], "is_correct_input() (datafusion.input.base.baseinputsource method)": [[6, "datafusion.input.base.BaseInputSource.is_correct_input", false]], "is_correct_input() (datafusion.input.location.locationinputplugin method)": [[8, "datafusion.input.location.LocationInputPlugin.is_correct_input", false]], "is_current_row() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_current_row", false]], "is_following() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_following", false]], "is_not_null() (datafusion.expr method)": [[5, "datafusion.Expr.is_not_null", false]], "is_not_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.is_not_null", false]], "is_null() (datafusion.expr method)": [[5, "datafusion.Expr.is_null", false]], "is_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.is_null", false]], "is_preceding() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_preceding", false]], "is_unbounded() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_unbounded", false]], "isfalse (in module datafusion.expr)": [[3, "datafusion.expr.IsFalse", false]], "isnan() (in module datafusion.functions)": [[4, "datafusion.functions.isnan", false]], "isnotfalse (in module datafusion.expr)": [[3, "datafusion.expr.IsNotFalse", false]], "isnotnull (in module datafusion.expr)": [[3, "datafusion.expr.IsNotNull", false]], "isnottrue (in module datafusion.expr)": [[3, "datafusion.expr.IsNotTrue", false]], "isnotunknown (in module datafusion.expr)": [[3, "datafusion.expr.IsNotUnknown", false]], "isnull (in module datafusion.expr)": [[3, "datafusion.expr.IsNull", false]], "istrue (in module datafusion.expr)": [[3, "datafusion.expr.IsTrue", false]], "isunknown (in module datafusion.expr)": [[3, "datafusion.expr.IsUnknown", false]], "iszero() (in module datafusion.functions)": [[4, "datafusion.functions.iszero", false]], "join (in module datafusion.expr)": [[3, "datafusion.expr.Join", false]], "join() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.join", false]], "join_on() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.join_on", false]], "joinconstraint (in module datafusion.expr)": [[3, "datafusion.expr.JoinConstraint", false]], "jointype (in module datafusion.expr)": [[3, "datafusion.expr.JoinType", false]], "kind (datafusion.catalog.table property)": [[0, "datafusion.catalog.Table.kind", false]], "kind (datafusion.table property)": [[5, "datafusion.Table.kind", false]], "lag() (in module datafusion.functions)": [[4, "datafusion.functions.lag", false]], "last_value() (in module datafusion.functions)": [[4, "datafusion.functions.last_value", false]], "lcm() (in module datafusion.functions)": [[4, "datafusion.functions.lcm", false]], "lead() (in module datafusion.functions)": [[4, "datafusion.functions.lead", false]], "left() (in module datafusion.functions)": [[4, "datafusion.functions.left", false]], "length() (in module datafusion.functions)": [[4, "datafusion.functions.length", false]], "levenshtein() (in module datafusion.functions)": [[4, "datafusion.functions.levenshtein", false]], "like (in module datafusion.expr)": [[3, "datafusion.expr.Like", false]], "limit (in module datafusion.expr)": [[3, "datafusion.expr.Limit", false]], "limit() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.limit", false]], "list_append() (in module datafusion.functions)": [[4, "datafusion.functions.list_append", false]], "list_cat() (in module datafusion.functions)": [[4, "datafusion.functions.list_cat", false]], "list_concat() (in module datafusion.functions)": [[4, "datafusion.functions.list_concat", false]], "list_dims() (in module datafusion.functions)": [[4, "datafusion.functions.list_dims", false]], "list_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.list_distinct", false]], "list_element() (in module datafusion.functions)": [[4, "datafusion.functions.list_element", false]], "list_except() (in module datafusion.functions)": [[4, "datafusion.functions.list_except", false]], "list_extract() (in module datafusion.functions)": [[4, "datafusion.functions.list_extract", false]], "list_indexof() (in module datafusion.functions)": [[4, "datafusion.functions.list_indexof", false]], "list_intersect() (in module datafusion.functions)": [[4, "datafusion.functions.list_intersect", false]], "list_join() (in module datafusion.functions)": [[4, "datafusion.functions.list_join", false]], "list_length() (in module datafusion.functions)": [[4, "datafusion.functions.list_length", false]], "list_ndims() (in module datafusion.functions)": [[4, "datafusion.functions.list_ndims", false]], "list_position() (in module datafusion.functions)": [[4, "datafusion.functions.list_position", false]], "list_positions() (in module datafusion.functions)": [[4, "datafusion.functions.list_positions", false]], "list_prepend() (in module datafusion.functions)": [[4, "datafusion.functions.list_prepend", false]], "list_push_back() (in module datafusion.functions)": [[4, "datafusion.functions.list_push_back", false]], "list_push_front() (in module datafusion.functions)": [[4, "datafusion.functions.list_push_front", false]], "list_remove() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove", false]], "list_remove_all() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove_all", false]], "list_remove_n() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove_n", false]], "list_repeat() (in module datafusion.functions)": [[4, "datafusion.functions.list_repeat", false]], "list_replace() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace", false]], "list_replace_all() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace_all", false]], "list_replace_n() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace_n", false]], "list_resize() (in module datafusion.functions)": [[4, "datafusion.functions.list_resize", false]], "list_slice() (in module datafusion.functions)": [[4, "datafusion.functions.list_slice", false]], "list_sort() (in module datafusion.functions)": [[4, "datafusion.functions.list_sort", false]], "list_to_string() (in module datafusion.functions)": [[4, "datafusion.functions.list_to_string", false]], "list_union() (in module datafusion.functions)": [[4, "datafusion.functions.list_union", false]], "lit() (in module datafusion)": [[5, "datafusion.lit", false]], "literal (in module datafusion.expr)": [[3, "datafusion.expr.Literal", false]], "literal() (datafusion.expr static method)": [[5, "datafusion.Expr.literal", false]], "literal() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.literal", false]], "literal() (in module datafusion)": [[5, "datafusion.literal", false]], "ln() (in module datafusion.functions)": [[4, "datafusion.functions.ln", false]], "localfilesystem (in module datafusion.object_store)": [[9, "datafusion.object_store.LocalFileSystem", false]], "locationinputplugin (class in datafusion.input.location)": [[8, "datafusion.input.location.LocationInputPlugin", false]], "log() (in module datafusion.functions)": [[4, "datafusion.functions.log", false]], "log10() (in module datafusion.functions)": [[4, "datafusion.functions.log10", false]], "log2() (in module datafusion.functions)": [[4, "datafusion.functions.log2", false]], "logical_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.logical_plan", false]], "logicalplan (class in datafusion)": [[5, "datafusion.LogicalPlan", false]], "logicalplan (class in datafusion.plan)": [[10, "datafusion.plan.LogicalPlan", false]], "lower() (in module datafusion.functions)": [[4, "datafusion.functions.lower", false]], "lpad() (in module datafusion.functions)": [[4, "datafusion.functions.lpad", false]], "ltrim() (in module datafusion.functions)": [[4, "datafusion.functions.ltrim", false]], "lz4 (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.LZ4", false]], "lz4_raw (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.LZ4_RAW", false]], "make_array() (in module datafusion.functions)": [[4, "datafusion.functions.make_array", false]], "make_date() (in module datafusion.functions)": [[4, "datafusion.functions.make_date", false]], "make_list() (in module datafusion.functions)": [[4, "datafusion.functions.make_list", false]], "max() (in module datafusion.functions)": [[4, "datafusion.functions.max", false]], "md5() (in module datafusion.functions)": [[4, "datafusion.functions.md5", false]], "mean() (in module datafusion.functions)": [[4, "datafusion.functions.mean", false]], "median() (in module datafusion.functions)": [[4, "datafusion.functions.median", false]], "memoize() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.memoize", false]], "merge() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.merge", false]], "merge() (datafusion.udf.accumulator method)": [[13, "datafusion.udf.Accumulator.merge", false]], "microsoftazure (in module datafusion.object_store)": [[9, "datafusion.object_store.MicrosoftAzure", false]], "min() (in module datafusion.functions)": [[4, "datafusion.functions.min", false]], "module": [[0, "module-datafusion.catalog", false], [1, "module-datafusion.context", false], [2, "module-datafusion.dataframe", false], [3, "module-datafusion.expr", false], [4, "module-datafusion.functions", false], [5, "module-datafusion", false], [6, "module-datafusion.input.base", false], [7, "module-datafusion.input", false], [8, "module-datafusion.input.location", false], [9, "module-datafusion.object_store", false], [10, "module-datafusion.plan", false], [11, "module-datafusion.record_batch", false], [12, "module-datafusion.substrait", false], [13, "module-datafusion.udf", false]], "named_struct() (in module datafusion.functions)": [[4, "datafusion.functions.named_struct", false]], "names() (datafusion.catalog method)": [[5, "datafusion.Catalog.names", false]], "names() (datafusion.catalog.catalog method)": [[0, "datafusion.catalog.Catalog.names", false]], "names() (datafusion.catalog.database method)": [[0, "datafusion.catalog.Database.names", false]], "names() (datafusion.database method)": [[5, "datafusion.Database.names", false]], "nanvl() (in module datafusion.functions)": [[4, "datafusion.functions.nanvl", false]], "negative (in module datafusion.expr)": [[3, "datafusion.expr.Negative", false]], "next() (datafusion.record_batch.recordbatchstream method)": [[11, "datafusion.record_batch.RecordBatchStream.next", false]], "next() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.next", false]], "not (in module datafusion.expr)": [[3, "datafusion.expr.Not", false]], "now() (in module datafusion.functions)": [[4, "datafusion.functions.now", false]], "nth_value() (in module datafusion.functions)": [[4, "datafusion.functions.nth_value", false]], "ntile() (in module datafusion.functions)": [[4, "datafusion.functions.ntile", false]], "null_treatment() (datafusion.expr method)": [[5, "datafusion.Expr.null_treatment", false]], "null_treatment() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.null_treatment", false]], "nullif() (in module datafusion.functions)": [[4, "datafusion.functions.nullif", false]], "nulls_first() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.nulls_first", false]], "nvl() (in module datafusion.functions)": [[4, "datafusion.functions.nvl", false]], "octet_length() (in module datafusion.functions)": [[4, "datafusion.functions.octet_length", false]], "optimized_logical_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.optimized_logical_plan", false]], "options_internal (datafusion.context.sqloptions attribute)": [[1, "datafusion.context.SQLOptions.options_internal", false]], "options_internal (datafusion.sqloptions attribute)": [[5, "datafusion.SQLOptions.options_internal", false]], "order_by() (datafusion.expr method)": [[5, "datafusion.Expr.order_by", false]], "order_by() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.order_by", false]], "order_by() (in module datafusion.functions)": [[4, "datafusion.functions.order_by", false]], "otherwise() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.otherwise", false]], "over() (datafusion.expr method)": [[5, "datafusion.Expr.over", false]], "over() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.over", false]], "overlay() (in module datafusion.functions)": [[4, "datafusion.functions.overlay", false]], "partition_by() (datafusion.expr method)": [[5, "datafusion.Expr.partition_by", false]], "partition_by() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.partition_by", false]], "partition_count (datafusion.executionplan property)": [[5, "datafusion.ExecutionPlan.partition_count", false]], "partition_count (datafusion.plan.executionplan property)": [[10, "datafusion.plan.ExecutionPlan.partition_count", false]], "partitioning (in module datafusion.expr)": [[3, "datafusion.expr.Partitioning", false]], "percent_rank() (in module datafusion.functions)": [[4, "datafusion.functions.percent_rank", false]], "pi() (in module datafusion.functions)": [[4, "datafusion.functions.pi", false]], "placeholder (in module datafusion.expr)": [[3, "datafusion.expr.Placeholder", false]], "plan (class in datafusion.substrait)": [[12, "datafusion.substrait.Plan", false]], "plan_internal (datafusion.substrait.plan attribute)": [[12, "datafusion.substrait.Plan.plan_internal", false]], "pow() (in module datafusion.functions)": [[4, "datafusion.functions.pow", false]], "power() (in module datafusion.functions)": [[4, "datafusion.functions.power", false]], "producer (class in datafusion.substrait)": [[12, "datafusion.substrait.Producer", false]], "projection (in module datafusion.expr)": [[3, "datafusion.expr.Projection", false]], "python_value() (datafusion.expr method)": [[5, "datafusion.Expr.python_value", false]], "python_value() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.python_value", false]], "radians() (in module datafusion.functions)": [[4, "datafusion.functions.radians", false]], "random() (in module datafusion.functions)": [[4, "datafusion.functions.random", false]], "range() (in module datafusion.functions)": [[4, "datafusion.functions.range", false]], "rank() (in module datafusion.functions)": [[4, "datafusion.functions.rank", false]], "raw_sort (datafusion.expr.sortexpr attribute)": [[3, "datafusion.expr.SortExpr.raw_sort", false]], "rbs (datafusion.record_batch.recordbatchstream attribute)": [[11, "datafusion.record_batch.RecordBatchStream.rbs", false]], "rbs (datafusion.recordbatchstream attribute)": [[5, "datafusion.RecordBatchStream.rbs", false]], "read_avro() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_avro", false]], "read_csv() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_csv", false]], "read_json() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_json", false]], "read_parquet() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_parquet", false]], "read_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_table", false]], "record_batch (datafusion.record_batch.recordbatch attribute)": [[11, "datafusion.record_batch.RecordBatch.record_batch", false]], "record_batch (datafusion.recordbatch attribute)": [[5, "datafusion.RecordBatch.record_batch", false]], "recordbatch (class in datafusion)": [[5, "datafusion.RecordBatch", false]], "recordbatch (class in datafusion.record_batch)": [[11, "datafusion.record_batch.RecordBatch", false]], "recordbatchstream (class in datafusion)": [[5, "datafusion.RecordBatchStream", false]], "recordbatchstream (class in datafusion.record_batch)": [[11, "datafusion.record_batch.RecordBatchStream", false]], "regexp_like() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_like", false]], "regexp_match() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_match", false]], "regexp_replace() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_replace", false]], "register_avro() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_avro", false]], "register_csv() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_csv", false]], "register_dataset() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_dataset", false]], "register_json() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_json", false]], "register_listing_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_listing_table", false]], "register_object_store() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_object_store", false]], "register_parquet() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_parquet", false]], "register_record_batches() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_record_batches", false]], "register_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_table", false]], "register_table_provider() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_table_provider", false]], "register_udaf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udaf", false]], "register_udf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udf", false]], "register_udwf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udwf", false]], "regr_avgx() (in module datafusion.functions)": [[4, "datafusion.functions.regr_avgx", false]], "regr_avgy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_avgy", false]], "regr_count() (in module datafusion.functions)": [[4, "datafusion.functions.regr_count", false]], "regr_intercept() (in module datafusion.functions)": [[4, "datafusion.functions.regr_intercept", false]], "regr_r2() (in module datafusion.functions)": [[4, "datafusion.functions.regr_r2", false]], "regr_slope() (in module datafusion.functions)": [[4, "datafusion.functions.regr_slope", false]], "regr_sxx() (in module datafusion.functions)": [[4, "datafusion.functions.regr_sxx", false]], "regr_sxy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_sxy", false]], "regr_syy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_syy", false]], "repartition (in module datafusion.expr)": [[3, "datafusion.expr.Repartition", false]], "repartition() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.repartition", false]], "repartition_by_hash() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.repartition_by_hash", false]], "repeat() (in module datafusion.functions)": [[4, "datafusion.functions.repeat", false]], "replace() (in module datafusion.functions)": [[4, "datafusion.functions.replace", false]], "reverse() (in module datafusion.functions)": [[4, "datafusion.functions.reverse", false]], "rex_call_operands() (datafusion.expr method)": [[5, "datafusion.Expr.rex_call_operands", false]], "rex_call_operands() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_call_operands", false]], "rex_call_operator() (datafusion.expr method)": [[5, "datafusion.Expr.rex_call_operator", false]], "rex_call_operator() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_call_operator", false]], "rex_type() (datafusion.expr method)": [[5, "datafusion.Expr.rex_type", false]], "rex_type() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_type", false]], "right() (in module datafusion.functions)": [[4, "datafusion.functions.right", false]], "round() (in module datafusion.functions)": [[4, "datafusion.functions.round", false]], "row_number() (in module datafusion.functions)": [[4, "datafusion.functions.row_number", false]], "rpad() (in module datafusion.functions)": [[4, "datafusion.functions.rpad", false]], "rtrim() (in module datafusion.functions)": [[4, "datafusion.functions.rtrim", false]], "runtimeconfig (class in datafusion.context)": [[1, "datafusion.context.RuntimeConfig", false]], "runtimeenvbuilder (class in datafusion)": [[5, "datafusion.RuntimeEnvBuilder", false]], "runtimeenvbuilder (class in datafusion.context)": [[1, "datafusion.context.RuntimeEnvBuilder", false]], "scalarsubquery (in module datafusion.expr)": [[3, "datafusion.expr.ScalarSubquery", false]], "scalarudf (class in datafusion)": [[5, "datafusion.ScalarUDF", false]], "scalarudf (class in datafusion.udf)": [[13, "datafusion.udf.ScalarUDF", false]], "scalarvariable (in module datafusion.expr)": [[3, "datafusion.expr.ScalarVariable", false]], "schema() (datafusion.catalog.table method)": [[0, "datafusion.catalog.Table.schema", false]], "schema() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.schema", false]], "schema() (datafusion.table method)": [[5, "datafusion.Table.schema", false]], "schema_name() (datafusion.expr method)": [[5, "datafusion.Expr.schema_name", false]], "schema_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.schema_name", false]], "select() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.select", false]], "select_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.select_columns", false]], "serde (class in datafusion.substrait)": [[12, "datafusion.substrait.Serde", false]], "serialize() (datafusion.substrait.serde static method)": [[12, "datafusion.substrait.Serde.serialize", false]], "serialize_bytes() (datafusion.substrait.serde static method)": [[12, "datafusion.substrait.Serde.serialize_bytes", false]], "serialize_to_plan() (datafusion.substrait.serde static method)": [[12, "datafusion.substrait.Serde.serialize_to_plan", false]], "session_id() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.session_id", false]], "sessionconfig (class in datafusion)": [[5, "datafusion.SessionConfig", false]], "sessionconfig (class in datafusion.context)": [[1, "datafusion.context.SessionConfig", false]], "sessioncontext (class in datafusion.context)": [[1, "datafusion.context.SessionContext", false]], "set() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.set", false]], "set() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.set", false]], "sha224() (in module datafusion.functions)": [[4, "datafusion.functions.sha224", false]], "sha256() (in module datafusion.functions)": [[4, "datafusion.functions.sha256", false]], "sha384() (in module datafusion.functions)": [[4, "datafusion.functions.sha384", false]], "sha512() (in module datafusion.functions)": [[4, "datafusion.functions.sha512", false]], "show() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.show", false]], "signum() (in module datafusion.functions)": [[4, "datafusion.functions.signum", false]], "similarto (in module datafusion.expr)": [[3, "datafusion.expr.SimilarTo", false]], "sin() (in module datafusion.functions)": [[4, "datafusion.functions.sin", false]], "sinh() (in module datafusion.functions)": [[4, "datafusion.functions.sinh", false]], "snappy (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.SNAPPY", false]], "sort (in module datafusion.expr)": [[3, "datafusion.expr.Sort", false]], "sort() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.sort", false]], "sort() (datafusion.expr method)": [[5, "datafusion.Expr.sort", false]], "sort() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.sort", false]], "sortexpr (class in datafusion.expr)": [[3, "datafusion.expr.SortExpr", false]], "split_part() (in module datafusion.functions)": [[4, "datafusion.functions.split_part", false]], "sql() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.sql", false]], "sql_with_options() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.sql_with_options", false]], "sqloptions (class in datafusion)": [[5, "datafusion.SQLOptions", false]], "sqloptions (class in datafusion.context)": [[1, "datafusion.context.SQLOptions", false]], "sqrt() (in module datafusion.functions)": [[4, "datafusion.functions.sqrt", false]], "stable (datafusion.udf.volatility attribute)": [[13, "datafusion.udf.Volatility.Stable", false]], "starts_with() (in module datafusion.functions)": [[4, "datafusion.functions.starts_with", false]], "state() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.state", false]], "state() (datafusion.udf.accumulator method)": [[13, "datafusion.udf.Accumulator.state", false]], "stddev() (in module datafusion.functions)": [[4, "datafusion.functions.stddev", false]], "stddev_pop() (in module datafusion.functions)": [[4, "datafusion.functions.stddev_pop", false]], "stddev_samp() (in module datafusion.functions)": [[4, "datafusion.functions.stddev_samp", false]], "string_agg() (in module datafusion.functions)": [[4, "datafusion.functions.string_agg", false]], "string_literal() (datafusion.expr static method)": [[5, "datafusion.Expr.string_literal", false]], "string_literal() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.string_literal", false]], "strpos() (in module datafusion.functions)": [[4, "datafusion.functions.strpos", false]], "struct() (in module datafusion.functions)": [[4, "datafusion.functions.struct", false]], "subquery (in module datafusion.expr)": [[3, "datafusion.expr.Subquery", false]], "subqueryalias (in module datafusion.expr)": [[3, "datafusion.expr.SubqueryAlias", false]], "substr() (in module datafusion.functions)": [[4, "datafusion.functions.substr", false]], "substr_index() (in module datafusion.functions)": [[4, "datafusion.functions.substr_index", false]], "substring() (in module datafusion.functions)": [[4, "datafusion.functions.substring", false]], "sum() (in module datafusion.functions)": [[4, "datafusion.functions.sum", false]], "supports_bounded_execution() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.supports_bounded_execution", false]], "table (class in datafusion)": [[5, "datafusion.Table", false]], "table (class in datafusion.catalog)": [[0, "datafusion.catalog.Table", false]], "table (datafusion.catalog.table attribute)": [[0, "datafusion.catalog.Table.table", false]], "table (datafusion.table attribute)": [[5, "datafusion.Table.table", false]], "table() (datafusion.catalog.database method)": [[0, "datafusion.catalog.Database.table", false]], "table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.table", false]], "table() (datafusion.database method)": [[5, "datafusion.Database.table", false]], "table_exist() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.table_exist", false]], "tableproviderexportable (class in datafusion.context)": [[1, "datafusion.context.TableProviderExportable", false]], "tablescan (in module datafusion.expr)": [[3, "datafusion.expr.TableScan", false]], "tail() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.tail", false]], "tan() (in module datafusion.functions)": [[4, "datafusion.functions.tan", false]], "tanh() (in module datafusion.functions)": [[4, "datafusion.functions.tanh", false]], "to_arrow_table() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_arrow_table", false]], "to_hex() (in module datafusion.functions)": [[4, "datafusion.functions.to_hex", false]], "to_pandas() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pandas", false]], "to_polars() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_polars", false]], "to_proto() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.to_proto", false]], "to_proto() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.to_proto", false]], "to_proto() (datafusion.plan.executionplan method)": [[10, "datafusion.plan.ExecutionPlan.to_proto", false]], "to_proto() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.to_proto", false]], "to_pyarrow() (datafusion.record_batch.recordbatch method)": [[11, "datafusion.record_batch.RecordBatch.to_pyarrow", false]], "to_pyarrow() (datafusion.recordbatch method)": [[5, "datafusion.RecordBatch.to_pyarrow", false]], "to_pydict() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pydict", false]], "to_pylist() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pylist", false]], "to_substrait_plan() (datafusion.substrait.producer static method)": [[12, "datafusion.substrait.Producer.to_substrait_plan", false]], "to_timestamp() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp", false]], "to_timestamp_micros() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_micros", false]], "to_timestamp_millis() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_millis", false]], "to_timestamp_seconds() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_seconds", false]], "to_unixtime() (in module datafusion.functions)": [[4, "datafusion.functions.to_unixtime", false]], "to_variant() (datafusion.expr method)": [[5, "datafusion.Expr.to_variant", false]], "to_variant() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.to_variant", false]], "to_variant() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.to_variant", false]], "to_variant() (datafusion.plan.logicalplan method)": [[10, "datafusion.plan.LogicalPlan.to_variant", false]], "transform() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.transform", false]], "translate() (in module datafusion.functions)": [[4, "datafusion.functions.translate", false]], "trim() (in module datafusion.functions)": [[4, "datafusion.functions.trim", false]], "trunc() (in module datafusion.functions)": [[4, "datafusion.functions.trunc", false]], "trycast (in module datafusion.expr)": [[3, "datafusion.expr.TryCast", false]], "types() (datafusion.expr method)": [[5, "datafusion.Expr.types", false]], "types() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.types", false]], "udaf() (datafusion.aggregateudf static method)": [[5, "datafusion.AggregateUDF.udaf", false]], "udaf() (datafusion.udf.aggregateudf static method)": [[13, "datafusion.udf.AggregateUDF.udaf", false]], "udf() (datafusion.scalarudf static method)": [[5, "datafusion.ScalarUDF.udf", false]], "udf() (datafusion.udf.scalarudf static method)": [[13, "datafusion.udf.ScalarUDF.udf", false]], "udwf() (datafusion.udf.windowudf static method)": [[13, "datafusion.udf.WindowUDF.udwf", false]], "udwf() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF.udwf", false]], "uncompressed (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.UNCOMPRESSED", false]], "union (in module datafusion.expr)": [[3, "datafusion.expr.Union", false]], "union() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.union", false]], "union_distinct() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.union_distinct", false]], "unnest (in module datafusion.expr)": [[3, "datafusion.expr.Unnest", false]], "unnest_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.unnest_columns", false]], "unnestexpr (in module datafusion.expr)": [[3, "datafusion.expr.UnnestExpr", false]], "update() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.update", false]], "update() (datafusion.udf.accumulator method)": [[13, "datafusion.udf.Accumulator.update", false]], "upper() (in module datafusion.functions)": [[4, "datafusion.functions.upper", false]], "uses_window_frame() (datafusion.udf.windowevaluator method)": [[13, "datafusion.udf.WindowEvaluator.uses_window_frame", false]], "uuid() (in module datafusion.functions)": [[4, "datafusion.functions.uuid", false]], "var() (in module datafusion.functions)": [[4, "datafusion.functions.var", false]], "var_pop() (in module datafusion.functions)": [[4, "datafusion.functions.var_pop", false]], "var_samp() (in module datafusion.functions)": [[4, "datafusion.functions.var_samp", false]], "var_sample() (in module datafusion.functions)": [[4, "datafusion.functions.var_sample", false]], "variant_name() (datafusion.expr method)": [[5, "datafusion.Expr.variant_name", false]], "variant_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.variant_name", false]], "volatile (datafusion.udf.volatility attribute)": [[13, "datafusion.udf.Volatility.Volatile", false]], "volatility (class in datafusion.udf)": [[13, "datafusion.udf.Volatility", false]], "when() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.when", false]], "when() (in module datafusion.functions)": [[4, "datafusion.functions.when", false]], "window (class in datafusion.expr)": [[3, "datafusion.expr.Window", false]], "window() (in module datafusion.functions)": [[4, "datafusion.functions.window", false]], "window_frame (datafusion.expr.windowframe attribute)": [[3, "datafusion.expr.WindowFrame.window_frame", false]], "window_frame (datafusion.windowframe attribute)": [[5, "datafusion.WindowFrame.window_frame", false]], "window_frame() (datafusion.expr method)": [[5, "datafusion.Expr.window_frame", false]], "window_frame() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.window_frame", false]], "windowevaluator (class in datafusion.udf)": [[13, "datafusion.udf.WindowEvaluator", false]], "windowexpr (in module datafusion.expr)": [[3, "datafusion.expr.WindowExpr", false]], "windowframe (class in datafusion)": [[5, "datafusion.WindowFrame", false]], "windowframe (class in datafusion.expr)": [[3, "datafusion.expr.WindowFrame", false]], "windowframebound (class in datafusion.expr)": [[3, "datafusion.expr.WindowFrameBound", false]], "windowudf (class in datafusion)": [[5, "datafusion.WindowUDF", false]], "windowudf (class in datafusion.udf)": [[13, "datafusion.udf.WindowUDF", false]], "with_allow_ddl() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_ddl", false]], "with_allow_ddl() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_ddl", false]], "with_allow_dml() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_dml", false]], "with_allow_dml() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_dml", false]], "with_allow_statements() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_statements", false]], "with_allow_statements() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_statements", false]], "with_batch_size() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_batch_size", false]], "with_batch_size() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_batch_size", false]], "with_column() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_column", false]], "with_column_renamed() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_column_renamed", false]], "with_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_columns", false]], "with_create_default_catalog_and_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_create_default_catalog_and_schema", false]], "with_create_default_catalog_and_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_create_default_catalog_and_schema", false]], "with_default_catalog_and_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_default_catalog_and_schema", false]], "with_default_catalog_and_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_default_catalog_and_schema", false]], "with_disk_manager_disabled() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_disabled", false]], "with_disk_manager_disabled() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_disabled", false]], "with_disk_manager_os() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_os", false]], "with_disk_manager_os() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_os", false]], "with_disk_manager_specified() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_specified", false]], "with_disk_manager_specified() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_specified", false]], "with_fair_spill_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_fair_spill_pool", false]], "with_fair_spill_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_fair_spill_pool", false]], "with_greedy_memory_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_greedy_memory_pool", false]], "with_greedy_memory_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_greedy_memory_pool", false]], "with_information_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_information_schema", false]], "with_information_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_information_schema", false]], "with_parquet_pruning() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_parquet_pruning", false]], "with_parquet_pruning() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_parquet_pruning", false]], "with_repartition_aggregations() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_aggregations", false]], "with_repartition_aggregations() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_aggregations", false]], "with_repartition_file_min_size() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_file_min_size", false]], "with_repartition_file_min_size() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_file_min_size", false]], "with_repartition_file_scans() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_file_scans", false]], "with_repartition_file_scans() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_file_scans", false]], "with_repartition_joins() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_joins", false]], "with_repartition_joins() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_joins", false]], "with_repartition_sorts() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_sorts", false]], "with_repartition_sorts() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_sorts", false]], "with_repartition_windows() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_windows", false]], "with_repartition_windows() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_windows", false]], "with_target_partitions() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_target_partitions", false]], "with_target_partitions() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_target_partitions", false]], "with_temp_file_path() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_temp_file_path", false]], "with_temp_file_path() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_temp_file_path", false]], "with_unbounded_memory_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_unbounded_memory_pool", false]], "with_unbounded_memory_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_unbounded_memory_pool", false]], "write_csv() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_csv", false]], "write_json() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_json", false]], "write_parquet() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_parquet", false]], "zstd (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.ZSTD", false]]}, "objects": {"": [[5, 0, 0, "-", "datafusion"]], "datafusion": [[5, 1, 1, "", "Accumulator"], [5, 1, 1, "", "AggregateUDF"], [5, 1, 1, "", "Catalog"], [5, 4, 1, "", "DFSchema"], [5, 1, 1, "", "Database"], [5, 1, 1, "", "ExecutionPlan"], [5, 1, 1, "", "Expr"], [5, 1, 1, "", "LogicalPlan"], [5, 1, 1, "", "RecordBatch"], [5, 1, 1, "", "RecordBatchStream"], [5, 1, 1, "", "RuntimeEnvBuilder"], [5, 1, 1, "", "SQLOptions"], [5, 1, 1, "", "ScalarUDF"], [5, 1, 1, "", "SessionConfig"], [5, 1, 1, "", "Table"], [5, 1, 1, "", "WindowFrame"], [5, 1, 1, "", "WindowUDF"], [0, 0, 0, "-", "catalog"], [5, 6, 1, "", "col"], [5, 6, 1, "", "column"], [1, 0, 0, "-", "context"], [2, 0, 0, "-", "dataframe"], [3, 0, 0, "-", "expr"], [4, 0, 0, "-", "functions"], [7, 0, 0, "-", "input"], [5, 6, 1, "", "lit"], [5, 6, 1, "", "literal"], [9, 0, 0, "-", "object_store"], [10, 0, 0, "-", "plan"], [11, 0, 0, "-", "record_batch"], [12, 0, 0, "-", "substrait"], [13, 0, 0, "-", "udf"]], "datafusion.Accumulator": [[5, 2, 1, "", "evaluate"], [5, 2, 1, "", "merge"], [5, 2, 1, "", "state"], [5, 2, 1, "", "update"]], "datafusion.AggregateUDF": [[5, 2, 1, "", "__call__"], [5, 3, 1, "", "_udaf"], [5, 2, 1, "", "udaf"]], "datafusion.Catalog": [[5, 3, 1, "", "catalog"], [5, 2, 1, "", "database"], [5, 2, 1, "", "names"]], "datafusion.Database": [[5, 3, 1, "", "db"], [5, 2, 1, "", "names"], [5, 2, 1, "", "table"]], "datafusion.ExecutionPlan": [[5, 2, 1, "", "__repr__"], [5, 3, 1, "", "_raw_plan"], [5, 2, 1, "", "children"], [5, 2, 1, "", "display"], [5, 2, 1, "", "display_indent"], [5, 2, 1, "", "from_proto"], [5, 5, 1, "", "partition_count"], [5, 2, 1, "", "to_proto"]], "datafusion.Expr": [[5, 2, 1, "", "__add__"], [5, 2, 1, "", "__and__"], [5, 2, 1, "", "__eq__"], [5, 2, 1, "", "__ge__"], [5, 2, 1, "", "__getitem__"], [5, 2, 1, "", "__gt__"], [5, 2, 1, "", "__invert__"], [5, 2, 1, "", "__le__"], [5, 2, 1, "", "__lt__"], [5, 2, 1, "", "__mod__"], [5, 2, 1, "", "__mul__"], [5, 2, 1, "", "__ne__"], [5, 2, 1, "", "__or__"], [5, 3, 1, "", "__radd__"], [5, 3, 1, "", "__rand__"], [5, 2, 1, "", "__repr__"], [5, 2, 1, "", "__richcmp__"], [5, 3, 1, "", "__rmod__"], [5, 3, 1, "", "__rmul__"], [5, 3, 1, "", "__ror__"], [5, 3, 1, "", "__rsub__"], [5, 3, 1, "", "__rtruediv__"], [5, 2, 1, "", "__sub__"], [5, 2, 1, "", "__truediv__"], [5, 3, 1, "", "_to_pyarrow_types"], [5, 2, 1, "", "alias"], [5, 2, 1, "", "between"], [5, 2, 1, "", "canonical_name"], [5, 2, 1, "", "cast"], [5, 2, 1, "", "column"], [5, 2, 1, "", "column_name"], [5, 2, 1, "", "display_name"], [5, 2, 1, "", "distinct"], [5, 3, 1, "", "expr"], [5, 2, 1, "", "fill_nan"], [5, 2, 1, "", "fill_null"], [5, 2, 1, "", "filter"], [5, 2, 1, "", "is_not_null"], [5, 2, 1, "", "is_null"], [5, 2, 1, "", "literal"], [5, 2, 1, "", "null_treatment"], [5, 2, 1, "", "order_by"], [5, 2, 1, "", "over"], [5, 2, 1, "", "partition_by"], [5, 2, 1, "", "python_value"], [5, 2, 1, "", "rex_call_operands"], [5, 2, 1, "", "rex_call_operator"], [5, 2, 1, "", "rex_type"], [5, 2, 1, "", "schema_name"], [5, 2, 1, "", "sort"], [5, 2, 1, "", "string_literal"], [5, 2, 1, "", "to_variant"], [5, 2, 1, "", "types"], [5, 2, 1, "", "variant_name"], [5, 2, 1, "", "window_frame"]], "datafusion.LogicalPlan": [[5, 2, 1, "", "__repr__"], [5, 3, 1, "", "_raw_plan"], [5, 2, 1, "", "display"], [5, 2, 1, "", "display_graphviz"], [5, 2, 1, "", "display_indent"], [5, 2, 1, "", "display_indent_schema"], [5, 2, 1, "", "from_proto"], [5, 2, 1, "", "inputs"], [5, 2, 1, "", "to_proto"], [5, 2, 1, "", "to_variant"]], "datafusion.RecordBatch": [[5, 3, 1, "", "record_batch"], [5, 2, 1, "", "to_pyarrow"]], "datafusion.RecordBatchStream": [[5, 2, 1, "", "__aiter__"], [5, 2, 1, "", "__anext__"], [5, 2, 1, "", "__iter__"], [5, 2, 1, "", "__next__"], [5, 2, 1, "", "next"], [5, 3, 1, "", "rbs"]], "datafusion.RuntimeEnvBuilder": [[5, 3, 1, "", "config_internal"], [5, 2, 1, "", "with_disk_manager_disabled"], [5, 2, 1, "", "with_disk_manager_os"], [5, 2, 1, "", "with_disk_manager_specified"], [5, 2, 1, "", "with_fair_spill_pool"], [5, 2, 1, "", "with_greedy_memory_pool"], [5, 2, 1, "", "with_temp_file_path"], [5, 2, 1, "", "with_unbounded_memory_pool"]], "datafusion.SQLOptions": [[5, 3, 1, "", "options_internal"], [5, 2, 1, "", "with_allow_ddl"], [5, 2, 1, "", "with_allow_dml"], [5, 2, 1, "", "with_allow_statements"]], "datafusion.ScalarUDF": [[5, 2, 1, "", "__call__"], [5, 3, 1, "", "_udf"], [5, 2, 1, "", "udf"]], "datafusion.SessionConfig": [[5, 3, 1, "", "config_internal"], [5, 2, 1, "", "set"], [5, 2, 1, "", "with_batch_size"], [5, 2, 1, "", "with_create_default_catalog_and_schema"], [5, 2, 1, "", "with_default_catalog_and_schema"], [5, 2, 1, "", "with_information_schema"], [5, 2, 1, "", "with_parquet_pruning"], [5, 2, 1, "", "with_repartition_aggregations"], [5, 2, 1, "", "with_repartition_file_min_size"], [5, 2, 1, "", "with_repartition_file_scans"], [5, 2, 1, "", "with_repartition_joins"], [5, 2, 1, "", "with_repartition_sorts"], [5, 2, 1, "", "with_repartition_windows"], [5, 2, 1, "", "with_target_partitions"]], "datafusion.Table": [[5, 5, 1, "", "kind"], [5, 2, 1, "", "schema"], [5, 3, 1, "", "table"]], "datafusion.WindowFrame": [[5, 2, 1, "", "get_frame_units"], [5, 2, 1, "", "get_lower_bound"], [5, 2, 1, "", "get_upper_bound"], [5, 3, 1, "", "window_frame"]], "datafusion.WindowUDF": [[5, 2, 1, "", "__call__"], [5, 3, 1, "", "_udwf"], [5, 2, 1, "", "udwf"]], "datafusion.catalog": [[0, 1, 1, "", "Catalog"], [0, 1, 1, "", "Database"], [0, 1, 1, "", "Table"]], "datafusion.catalog.Catalog": [[0, 3, 1, "", "catalog"], [0, 2, 1, "", "database"], [0, 2, 1, "", "names"]], "datafusion.catalog.Database": [[0, 3, 1, "", "db"], [0, 2, 1, "", "names"], [0, 2, 1, "", "table"]], "datafusion.catalog.Table": [[0, 5, 1, "", "kind"], [0, 2, 1, "", "schema"], [0, 3, 1, "", "table"]], "datafusion.context": [[1, 1, 1, "", "ArrowArrayExportable"], [1, 1, 1, "", "ArrowStreamExportable"], [1, 1, 1, "", "RuntimeConfig"], [1, 1, 1, "", "RuntimeEnvBuilder"], [1, 1, 1, "", "SQLOptions"], [1, 1, 1, "", "SessionConfig"], [1, 1, 1, "", "SessionContext"], [1, 1, 1, "", "TableProviderExportable"]], "datafusion.context.ArrowArrayExportable": [[1, 2, 1, "", "__arrow_c_array__"]], "datafusion.context.ArrowStreamExportable": [[1, 2, 1, "", "__arrow_c_stream__"]], "datafusion.context.RuntimeEnvBuilder": [[1, 3, 1, "", "config_internal"], [1, 2, 1, "", "with_disk_manager_disabled"], [1, 2, 1, "", "with_disk_manager_os"], [1, 2, 1, "", "with_disk_manager_specified"], [1, 2, 1, "", "with_fair_spill_pool"], [1, 2, 1, "", "with_greedy_memory_pool"], [1, 2, 1, "", "with_temp_file_path"], [1, 2, 1, "", "with_unbounded_memory_pool"]], "datafusion.context.SQLOptions": [[1, 3, 1, "", "options_internal"], [1, 2, 1, "", "with_allow_ddl"], [1, 2, 1, "", "with_allow_dml"], [1, 2, 1, "", "with_allow_statements"]], "datafusion.context.SessionConfig": [[1, 3, 1, "", "config_internal"], [1, 2, 1, "", "set"], [1, 2, 1, "", "with_batch_size"], [1, 2, 1, "", "with_create_default_catalog_and_schema"], [1, 2, 1, "", "with_default_catalog_and_schema"], [1, 2, 1, "", "with_information_schema"], [1, 2, 1, "", "with_parquet_pruning"], [1, 2, 1, "", "with_repartition_aggregations"], [1, 2, 1, "", "with_repartition_file_min_size"], [1, 2, 1, "", "with_repartition_file_scans"], [1, 2, 1, "", "with_repartition_joins"], [1, 2, 1, "", "with_repartition_sorts"], [1, 2, 1, "", "with_repartition_windows"], [1, 2, 1, "", "with_target_partitions"]], "datafusion.context.SessionContext": [[1, 2, 1, "", "catalog"], [1, 2, 1, "", "create_dataframe"], [1, 2, 1, "", "create_dataframe_from_logical_plan"], [1, 3, 1, "", "ctx"], [1, 2, 1, "", "deregister_table"], [1, 2, 1, "", "empty_table"], [1, 2, 1, "", "enable_url_table"], [1, 2, 1, "", "execute"], [1, 2, 1, "", "from_arrow"], [1, 2, 1, "", "from_arrow_table"], [1, 2, 1, "", "from_pandas"], [1, 2, 1, "", "from_polars"], [1, 2, 1, "", "from_pydict"], [1, 2, 1, "", "from_pylist"], [1, 2, 1, "", "read_avro"], [1, 2, 1, "", "read_csv"], [1, 2, 1, "", "read_json"], [1, 2, 1, "", "read_parquet"], [1, 2, 1, "", "read_table"], [1, 2, 1, "", "register_avro"], [1, 2, 1, "", "register_csv"], [1, 2, 1, "", "register_dataset"], [1, 2, 1, "", "register_json"], [1, 2, 1, "", "register_listing_table"], [1, 2, 1, "", "register_object_store"], [1, 2, 1, "", "register_parquet"], [1, 2, 1, "", "register_record_batches"], [1, 2, 1, "", "register_table"], [1, 2, 1, "", "register_table_provider"], [1, 2, 1, "", "register_udaf"], [1, 2, 1, "", "register_udf"], [1, 2, 1, "", "register_udwf"], [1, 2, 1, "", "session_id"], [1, 2, 1, "", "sql"], [1, 2, 1, "", "sql_with_options"], [1, 2, 1, "", "table"], [1, 2, 1, "", "table_exist"]], "datafusion.context.TableProviderExportable": [[1, 2, 1, "", "__datafusion_table_provider__"]], "datafusion.dataframe": [[2, 1, 1, "", "Compression"], [2, 1, 1, "", "DataFrame"]], "datafusion.dataframe.Compression": [[2, 3, 1, "", "BROTLI"], [2, 3, 1, "", "GZIP"], [2, 3, 1, "", "LZ4"], [2, 3, 1, "", "LZ4_RAW"], [2, 3, 1, "", "SNAPPY"], [2, 3, 1, "", "UNCOMPRESSED"], [2, 3, 1, "", "ZSTD"], [2, 2, 1, "", "from_str"], [2, 2, 1, "", "get_default_level"]], "datafusion.dataframe.DataFrame": [[2, 2, 1, "", "__arrow_c_stream__"], [2, 2, 1, "", "__getitem__"], [2, 2, 1, "", "__repr__"], [2, 2, 1, "", "_repr_html_"], [2, 2, 1, "", "aggregate"], [2, 2, 1, "", "cache"], [2, 2, 1, "", "cast"], [2, 2, 1, "", "collect"], [2, 2, 1, "", "collect_partitioned"], [2, 2, 1, "", "count"], [2, 2, 1, "", "describe"], [2, 3, 1, "", "df"], [2, 2, 1, "", "distinct"], [2, 2, 1, "", "drop"], [2, 2, 1, "", "except_all"], [2, 2, 1, "", "execute_stream"], [2, 2, 1, "", "execute_stream_partitioned"], [2, 2, 1, "", "execution_plan"], [2, 2, 1, "", "explain"], [2, 2, 1, "", "filter"], [2, 2, 1, "", "head"], [2, 2, 1, "", "intersect"], [2, 2, 1, "", "join"], [2, 2, 1, "", "join_on"], [2, 2, 1, "", "limit"], [2, 2, 1, "", "logical_plan"], [2, 2, 1, "", "optimized_logical_plan"], [2, 2, 1, "", "repartition"], [2, 2, 1, "", "repartition_by_hash"], [2, 2, 1, "", "schema"], [2, 2, 1, "", "select"], [2, 2, 1, "", "select_columns"], [2, 2, 1, "", "show"], [2, 2, 1, "", "sort"], [2, 2, 1, "", "tail"], [2, 2, 1, "", "to_arrow_table"], [2, 2, 1, "", "to_pandas"], [2, 2, 1, "", "to_polars"], [2, 2, 1, "", "to_pydict"], [2, 2, 1, "", "to_pylist"], [2, 2, 1, "", "transform"], [2, 2, 1, "", "union"], [2, 2, 1, "", "union_distinct"], [2, 2, 1, "", "unnest_columns"], [2, 2, 1, "", "with_column"], [2, 2, 1, "", "with_column_renamed"], [2, 2, 1, "", "with_columns"], [2, 2, 1, "", "write_csv"], [2, 2, 1, "", "write_json"], [2, 2, 1, "", "write_parquet"]], "datafusion.expr": [[3, 4, 1, "", "Aggregate"], [3, 4, 1, "", "AggregateFunction"], [3, 4, 1, "", "Alias"], [3, 4, 1, "", "Analyze"], [3, 4, 1, "", "Between"], [3, 4, 1, "", "BinaryExpr"], [3, 4, 1, "", "Case"], [3, 1, 1, "", "CaseBuilder"], [3, 4, 1, "", "Cast"], [3, 4, 1, "", "Column"], [3, 4, 1, "", "CreateMemoryTable"], [3, 4, 1, "", "CreateView"], [3, 4, 1, "", "Distinct"], [3, 4, 1, "", "DropTable"], [3, 4, 1, "", "EmptyRelation"], [3, 4, 1, "", "Exists"], [3, 4, 1, "", "Explain"], [3, 1, 1, "", "Expr"], [3, 4, 1, "", "Extension"], [3, 4, 1, "", "Filter"], [3, 4, 1, "", "GroupingSet"], [3, 4, 1, "", "ILike"], [3, 4, 1, "", "InList"], [3, 4, 1, "", "InSubquery"], [3, 4, 1, "", "IsFalse"], [3, 4, 1, "", "IsNotFalse"], [3, 4, 1, "", "IsNotNull"], [3, 4, 1, "", "IsNotTrue"], [3, 4, 1, "", "IsNotUnknown"], [3, 4, 1, "", "IsNull"], [3, 4, 1, "", "IsTrue"], [3, 4, 1, "", "IsUnknown"], [3, 4, 1, "", "Join"], [3, 4, 1, "", "JoinConstraint"], [3, 4, 1, "", "JoinType"], [3, 4, 1, "", "Like"], [3, 4, 1, "", "Limit"], [3, 4, 1, "", "Literal"], [3, 4, 1, "", "Negative"], [3, 4, 1, "", "Not"], [3, 4, 1, "", "Partitioning"], [3, 4, 1, "", "Placeholder"], [3, 4, 1, "", "Projection"], [3, 4, 1, "", "Repartition"], [3, 4, 1, "", "ScalarSubquery"], [3, 4, 1, "", "ScalarVariable"], [3, 4, 1, "", "SimilarTo"], [3, 4, 1, "", "Sort"], [3, 1, 1, "", "SortExpr"], [3, 4, 1, "", "Subquery"], [3, 4, 1, "", "SubqueryAlias"], [3, 4, 1, "", "TableScan"], [3, 4, 1, "", "TryCast"], [3, 4, 1, "", "Union"], [3, 4, 1, "", "Unnest"], [3, 4, 1, "", "UnnestExpr"], [3, 1, 1, "", "Window"], [3, 4, 1, "", "WindowExpr"], [3, 1, 1, "", "WindowFrame"], [3, 1, 1, "", "WindowFrameBound"]], "datafusion.expr.CaseBuilder": [[3, 3, 1, "", "case_builder"], [3, 2, 1, "", "end"], [3, 2, 1, "", "otherwise"], [3, 2, 1, "", "when"]], "datafusion.expr.Expr": [[3, 2, 1, "", "__add__"], [3, 2, 1, "", "__and__"], [3, 2, 1, "", "__eq__"], [3, 2, 1, "", "__ge__"], [3, 2, 1, "", "__getitem__"], [3, 2, 1, "", "__gt__"], [3, 2, 1, "", "__invert__"], [3, 2, 1, "", "__le__"], [3, 2, 1, "", "__lt__"], [3, 2, 1, "", "__mod__"], [3, 2, 1, "", "__mul__"], [3, 2, 1, "", "__ne__"], [3, 2, 1, "", "__or__"], [3, 3, 1, "", "__radd__"], [3, 3, 1, "", "__rand__"], [3, 2, 1, "", "__repr__"], [3, 2, 1, "", "__richcmp__"], [3, 3, 1, "", "__rmod__"], [3, 3, 1, "", "__rmul__"], [3, 3, 1, "", "__ror__"], [3, 3, 1, "", "__rsub__"], [3, 3, 1, "", "__rtruediv__"], [3, 2, 1, "", "__sub__"], [3, 2, 1, "", "__truediv__"], [3, 3, 1, "", "_to_pyarrow_types"], [3, 2, 1, "", "alias"], [3, 2, 1, "", "between"], [3, 2, 1, "", "canonical_name"], [3, 2, 1, "", "cast"], [3, 2, 1, "", "column"], [3, 2, 1, "", "column_name"], [3, 2, 1, "", "display_name"], [3, 2, 1, "", "distinct"], [3, 3, 1, "", "expr"], [3, 2, 1, "", "fill_nan"], [3, 2, 1, "", "fill_null"], [3, 2, 1, "", "filter"], [3, 2, 1, "", "is_not_null"], [3, 2, 1, "", "is_null"], [3, 2, 1, "", "literal"], [3, 2, 1, "", "null_treatment"], [3, 2, 1, "", "order_by"], [3, 2, 1, "", "over"], [3, 2, 1, "", "partition_by"], [3, 2, 1, "", "python_value"], [3, 2, 1, "", "rex_call_operands"], [3, 2, 1, "", "rex_call_operator"], [3, 2, 1, "", "rex_type"], [3, 2, 1, "", "schema_name"], [3, 2, 1, "", "sort"], [3, 2, 1, "", "string_literal"], [3, 2, 1, "", "to_variant"], [3, 2, 1, "", "types"], [3, 2, 1, "", "variant_name"], [3, 2, 1, "", "window_frame"]], "datafusion.expr.SortExpr": [[3, 2, 1, "", "__repr__"], [3, 2, 1, "", "ascending"], [3, 2, 1, "", "expr"], [3, 2, 1, "", "nulls_first"], [3, 3, 1, "", "raw_sort"]], "datafusion.expr.Window": [[3, 3, 1, "", "_null_treatment"], [3, 3, 1, "", "_order_by"], [3, 3, 1, "", "_partition_by"], [3, 3, 1, "", "_window_frame"]], "datafusion.expr.WindowFrame": [[3, 2, 1, "", "get_frame_units"], [3, 2, 1, "", "get_lower_bound"], [3, 2, 1, "", "get_upper_bound"], [3, 3, 1, "", "window_frame"]], "datafusion.expr.WindowFrameBound": [[3, 3, 1, "", "frame_bound"], [3, 2, 1, "", "get_offset"], [3, 2, 1, "", "is_current_row"], [3, 2, 1, "", "is_following"], [3, 2, 1, "", "is_preceding"], [3, 2, 1, "", "is_unbounded"]], "datafusion.functions": [[4, 6, 1, "", "abs"], [4, 6, 1, "", "acos"], [4, 6, 1, "", "acosh"], [4, 6, 1, "", "alias"], [4, 6, 1, "", "approx_distinct"], [4, 6, 1, "", "approx_median"], [4, 6, 1, "", "approx_percentile_cont"], [4, 6, 1, "", "approx_percentile_cont_with_weight"], [4, 6, 1, "", "array"], [4, 6, 1, "", "array_agg"], [4, 6, 1, "", "array_append"], [4, 6, 1, "", "array_cat"], [4, 6, 1, "", "array_concat"], [4, 6, 1, "", "array_dims"], [4, 6, 1, "", "array_distinct"], [4, 6, 1, "", "array_element"], [4, 6, 1, "", "array_empty"], [4, 6, 1, "", "array_except"], [4, 6, 1, "", "array_extract"], [4, 6, 1, "", "array_has"], [4, 6, 1, "", "array_has_all"], [4, 6, 1, "", "array_has_any"], [4, 6, 1, "", "array_indexof"], [4, 6, 1, "", "array_intersect"], [4, 6, 1, "", "array_join"], [4, 6, 1, "", "array_length"], [4, 6, 1, "", "array_ndims"], [4, 6, 1, "", "array_pop_back"], [4, 6, 1, "", "array_pop_front"], [4, 6, 1, "", "array_position"], [4, 6, 1, "", "array_positions"], [4, 6, 1, "", "array_prepend"], [4, 6, 1, "", "array_push_back"], [4, 6, 1, "", "array_push_front"], [4, 6, 1, "", "array_remove"], [4, 6, 1, "", "array_remove_all"], [4, 6, 1, "", "array_remove_n"], [4, 6, 1, "", "array_repeat"], [4, 6, 1, "", "array_replace"], [4, 6, 1, "", "array_replace_all"], [4, 6, 1, "", "array_replace_n"], [4, 6, 1, "", "array_resize"], [4, 6, 1, "", "array_slice"], [4, 6, 1, "", "array_sort"], [4, 6, 1, "", "array_to_string"], [4, 6, 1, "", "array_union"], [4, 6, 1, "", "arrow_cast"], [4, 6, 1, "", "arrow_typeof"], [4, 6, 1, "", "ascii"], [4, 6, 1, "", "asin"], [4, 6, 1, "", "asinh"], [4, 6, 1, "", "atan"], [4, 6, 1, "", "atan2"], [4, 6, 1, "", "atanh"], [4, 6, 1, "", "avg"], [4, 6, 1, "", "bit_and"], [4, 6, 1, "", "bit_length"], [4, 6, 1, "", "bit_or"], [4, 6, 1, "", "bit_xor"], [4, 6, 1, "", "bool_and"], [4, 6, 1, "", "bool_or"], [4, 6, 1, "", "btrim"], [4, 6, 1, "", "cardinality"], [4, 6, 1, "", "case"], [4, 6, 1, "", "cbrt"], [4, 6, 1, "", "ceil"], [4, 6, 1, "", "char_length"], [4, 6, 1, "", "character_length"], [4, 6, 1, "", "chr"], [4, 6, 1, "", "coalesce"], [4, 6, 1, "", "col"], [4, 6, 1, "", "concat"], [4, 6, 1, "", "concat_ws"], [4, 6, 1, "", "corr"], [4, 6, 1, "", "cos"], [4, 6, 1, "", "cosh"], [4, 6, 1, "", "cot"], [4, 6, 1, "", "count"], [4, 6, 1, "", "count_star"], [4, 6, 1, "", "covar"], [4, 6, 1, "", "covar_pop"], [4, 6, 1, "", "covar_samp"], [4, 6, 1, "", "cume_dist"], [4, 6, 1, "", "current_date"], [4, 6, 1, "", "current_time"], [4, 6, 1, "", "date_bin"], [4, 6, 1, "", "date_part"], [4, 6, 1, "", "date_trunc"], [4, 6, 1, "", "datepart"], [4, 6, 1, "", "datetrunc"], [4, 6, 1, "", "decode"], [4, 6, 1, "", "degrees"], [4, 6, 1, "", "dense_rank"], [4, 6, 1, "", "digest"], [4, 6, 1, "", "empty"], [4, 6, 1, "", "encode"], [4, 6, 1, "", "ends_with"], [4, 6, 1, "", "exp"], [4, 6, 1, "", "extract"], [4, 6, 1, "", "factorial"], [4, 6, 1, "", "find_in_set"], [4, 6, 1, "", "first_value"], [4, 6, 1, "", "flatten"], [4, 6, 1, "", "floor"], [4, 6, 1, "", "from_unixtime"], [4, 6, 1, "", "gcd"], [4, 6, 1, "", "in_list"], [4, 6, 1, "", "initcap"], [4, 6, 1, "", "isnan"], [4, 6, 1, "", "iszero"], [4, 6, 1, "", "lag"], [4, 6, 1, "", "last_value"], [4, 6, 1, "", "lcm"], [4, 6, 1, "", "lead"], [4, 6, 1, "", "left"], [4, 6, 1, "", "length"], [4, 6, 1, "", "levenshtein"], [4, 6, 1, "", "list_append"], [4, 6, 1, "", "list_cat"], [4, 6, 1, "", "list_concat"], [4, 6, 1, "", "list_dims"], [4, 6, 1, "", "list_distinct"], [4, 6, 1, "", "list_element"], [4, 6, 1, "", "list_except"], [4, 6, 1, "", "list_extract"], [4, 6, 1, "", "list_indexof"], [4, 6, 1, "", "list_intersect"], [4, 6, 1, "", "list_join"], [4, 6, 1, "", "list_length"], [4, 6, 1, "", "list_ndims"], [4, 6, 1, "", "list_position"], [4, 6, 1, "", "list_positions"], [4, 6, 1, "", "list_prepend"], [4, 6, 1, "", "list_push_back"], [4, 6, 1, "", "list_push_front"], [4, 6, 1, "", "list_remove"], [4, 6, 1, "", "list_remove_all"], [4, 6, 1, "", "list_remove_n"], [4, 6, 1, "", "list_repeat"], [4, 6, 1, "", "list_replace"], [4, 6, 1, "", "list_replace_all"], [4, 6, 1, "", "list_replace_n"], [4, 6, 1, "", "list_resize"], [4, 6, 1, "", "list_slice"], [4, 6, 1, "", "list_sort"], [4, 6, 1, "", "list_to_string"], [4, 6, 1, "", "list_union"], [4, 6, 1, "", "ln"], [4, 6, 1, "", "log"], [4, 6, 1, "", "log10"], [4, 6, 1, "", "log2"], [4, 6, 1, "", "lower"], [4, 6, 1, "", "lpad"], [4, 6, 1, "", "ltrim"], [4, 6, 1, "", "make_array"], [4, 6, 1, "", "make_date"], [4, 6, 1, "", "make_list"], [4, 6, 1, "", "max"], [4, 6, 1, "", "md5"], [4, 6, 1, "", "mean"], [4, 6, 1, "", "median"], [4, 6, 1, "", "min"], [4, 6, 1, "", "named_struct"], [4, 6, 1, "", "nanvl"], [4, 6, 1, "", "now"], [4, 6, 1, "", "nth_value"], [4, 6, 1, "", "ntile"], [4, 6, 1, "", "nullif"], [4, 6, 1, "", "nvl"], [4, 6, 1, "", "octet_length"], [4, 6, 1, "", "order_by"], [4, 6, 1, "", "overlay"], [4, 6, 1, "", "percent_rank"], [4, 6, 1, "", "pi"], [4, 6, 1, "", "pow"], [4, 6, 1, "", "power"], [4, 6, 1, "", "radians"], [4, 6, 1, "", "random"], [4, 6, 1, "", "range"], [4, 6, 1, "", "rank"], [4, 6, 1, "", "regexp_like"], [4, 6, 1, "", "regexp_match"], [4, 6, 1, "", "regexp_replace"], [4, 6, 1, "", "regr_avgx"], [4, 6, 1, "", "regr_avgy"], [4, 6, 1, "", "regr_count"], [4, 6, 1, "", "regr_intercept"], [4, 6, 1, "", "regr_r2"], [4, 6, 1, "", "regr_slope"], [4, 6, 1, "", "regr_sxx"], [4, 6, 1, "", "regr_sxy"], [4, 6, 1, "", "regr_syy"], [4, 6, 1, "", "repeat"], [4, 6, 1, "", "replace"], [4, 6, 1, "", "reverse"], [4, 6, 1, "", "right"], [4, 6, 1, "", "round"], [4, 6, 1, "", "row_number"], [4, 6, 1, "", "rpad"], [4, 6, 1, "", "rtrim"], [4, 6, 1, "", "sha224"], [4, 6, 1, "", "sha256"], [4, 6, 1, "", "sha384"], [4, 6, 1, "", "sha512"], [4, 6, 1, "", "signum"], [4, 6, 1, "", "sin"], [4, 6, 1, "", "sinh"], [4, 6, 1, "", "split_part"], [4, 6, 1, "", "sqrt"], [4, 6, 1, "", "starts_with"], [4, 6, 1, "", "stddev"], [4, 6, 1, "", "stddev_pop"], [4, 6, 1, "", "stddev_samp"], [4, 6, 1, "", "string_agg"], [4, 6, 1, "", "strpos"], [4, 6, 1, "", "struct"], [4, 6, 1, "", "substr"], [4, 6, 1, "", "substr_index"], [4, 6, 1, "", "substring"], [4, 6, 1, "", "sum"], [4, 6, 1, "", "tan"], [4, 6, 1, "", "tanh"], [4, 6, 1, "", "to_hex"], [4, 6, 1, "", "to_timestamp"], [4, 6, 1, "", "to_timestamp_micros"], [4, 6, 1, "", "to_timestamp_millis"], [4, 6, 1, "", "to_timestamp_seconds"], [4, 6, 1, "", "to_unixtime"], [4, 6, 1, "", "translate"], [4, 6, 1, "", "trim"], [4, 6, 1, "", "trunc"], [4, 6, 1, "", "upper"], [4, 6, 1, "", "uuid"], [4, 6, 1, "", "var"], [4, 6, 1, "", "var_pop"], [4, 6, 1, "", "var_samp"], [4, 6, 1, "", "var_sample"], [4, 6, 1, "", "when"], [4, 6, 1, "", "window"]], "datafusion.input": [[6, 0, 0, "-", "base"], [8, 0, 0, "-", "location"]], "datafusion.input.base": [[6, 1, 1, "", "BaseInputSource"]], "datafusion.input.base.BaseInputSource": [[6, 2, 1, "", "build_table"], [6, 2, 1, "", "is_correct_input"]], "datafusion.input.location": [[8, 1, 1, "", "LocationInputPlugin"]], "datafusion.input.location.LocationInputPlugin": [[8, 2, 1, "", "build_table"], [8, 2, 1, "", "is_correct_input"]], "datafusion.object_store": [[9, 4, 1, "", "AmazonS3"], [9, 4, 1, "", "GoogleCloud"], [9, 4, 1, "", "Http"], [9, 4, 1, "", "LocalFileSystem"], [9, 4, 1, "", "MicrosoftAzure"]], "datafusion.plan": [[10, 1, 1, "", "ExecutionPlan"], [10, 1, 1, "", "LogicalPlan"]], "datafusion.plan.ExecutionPlan": [[10, 2, 1, "", "__repr__"], [10, 3, 1, "", "_raw_plan"], [10, 2, 1, "", "children"], [10, 2, 1, "", "display"], [10, 2, 1, "", "display_indent"], [10, 2, 1, "", "from_proto"], [10, 5, 1, "", "partition_count"], [10, 2, 1, "", "to_proto"]], "datafusion.plan.LogicalPlan": [[10, 2, 1, "", "__repr__"], [10, 3, 1, "", "_raw_plan"], [10, 2, 1, "", "display"], [10, 2, 1, "", "display_graphviz"], [10, 2, 1, "", "display_indent"], [10, 2, 1, "", "display_indent_schema"], [10, 2, 1, "", "from_proto"], [10, 2, 1, "", "inputs"], [10, 2, 1, "", "to_proto"], [10, 2, 1, "", "to_variant"]], "datafusion.record_batch": [[11, 1, 1, "", "RecordBatch"], [11, 1, 1, "", "RecordBatchStream"]], "datafusion.record_batch.RecordBatch": [[11, 3, 1, "", "record_batch"], [11, 2, 1, "", "to_pyarrow"]], "datafusion.record_batch.RecordBatchStream": [[11, 2, 1, "", "__aiter__"], [11, 2, 1, "", "__anext__"], [11, 2, 1, "", "__iter__"], [11, 2, 1, "", "__next__"], [11, 2, 1, "", "next"], [11, 3, 1, "", "rbs"]], "datafusion.substrait": [[12, 1, 1, "", "Consumer"], [12, 1, 1, "", "Plan"], [12, 1, 1, "", "Producer"], [12, 1, 1, "", "Serde"]], "datafusion.substrait.Consumer": [[12, 2, 1, "", "from_substrait_plan"]], "datafusion.substrait.Plan": [[12, 2, 1, "", "encode"], [12, 3, 1, "", "plan_internal"]], "datafusion.substrait.Producer": [[12, 2, 1, "", "to_substrait_plan"]], "datafusion.substrait.Serde": [[12, 2, 1, "", "deserialize"], [12, 2, 1, "", "deserialize_bytes"], [12, 2, 1, "", "serialize"], [12, 2, 1, "", "serialize_bytes"], [12, 2, 1, "", "serialize_to_plan"]], "datafusion.udf": [[13, 1, 1, "", "Accumulator"], [13, 1, 1, "", "AggregateUDF"], [13, 1, 1, "", "ScalarUDF"], [13, 1, 1, "", "Volatility"], [13, 1, 1, "", "WindowEvaluator"], [13, 1, 1, "", "WindowUDF"], [13, 4, 1, "", "_R"], [13, 4, 1, "", "_W"]], "datafusion.udf.Accumulator": [[13, 2, 1, "", "evaluate"], [13, 2, 1, "", "merge"], [13, 2, 1, "", "state"], [13, 2, 1, "", "update"]], "datafusion.udf.AggregateUDF": [[13, 2, 1, "", "__call__"], [13, 3, 1, "", "_udaf"], [13, 2, 1, "", "udaf"]], "datafusion.udf.ScalarUDF": [[13, 2, 1, "", "__call__"], [13, 3, 1, "", "_udf"], [13, 2, 1, "", "udf"]], "datafusion.udf.Volatility": [[13, 3, 1, "", "Immutable"], [13, 3, 1, "", "Stable"], [13, 3, 1, "", "Volatile"], [13, 2, 1, "", "__str__"]], "datafusion.udf.WindowEvaluator": [[13, 2, 1, "", "evaluate"], [13, 2, 1, "", "evaluate_all"], [13, 2, 1, "", "evaluate_all_with_rank"], [13, 2, 1, "", "get_range"], [13, 2, 1, "", "include_rank"], [13, 2, 1, "", "is_causal"], [13, 2, 1, "", "memoize"], [13, 2, 1, "", "supports_bounded_execution"], [13, 2, 1, "", "uses_window_frame"]], "datafusion.udf.WindowUDF": [[13, 2, 1, "", "__call__"], [13, 3, 1, "", "_udwf"], [13, 2, 1, "", "udwf"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "data", "Python data"], "5": ["py", "property", "Python property"], "6": ["py", "function", "Python function"]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:attribute", "4": "py:data", "5": "py:property", "6": "py:function"}, "terms": {"": [1, 2, 3, 4, 5, 13, 15, 20, 23, 25, 26, 27], "0": [2, 3, 4, 5, 13, 17, 18, 19, 20, 21, 24, 25, 26, 28, 29, 36, 37], "01": [17, 21, 24], "01t00": 21, "02": 21, "04023": 4, "05": [21, 24], "06": [21, 24], "08": 21, "09": [17, 21], "1": [2, 3, 4, 5, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "10": [4, 5, 13, 16, 17, 18, 20, 21, 24, 25, 26, 28, 29, 37], "100": [4, 16, 17, 18, 26, 28, 29], "1000": [1, 19], "10000000": 27, "1024": [1, 5], "103": [16, 18, 28, 29], "104": [16, 28, 29], "105": [16, 18, 28, 29], "107": 18, "109": [16, 28, 29, 37], "11": [16, 17, 18, 19, 20, 21, 25, 26, 28, 29], "110": 18, "111": [16, 28, 29], "112": 18, "115": [16, 18, 28, 29], "12": [15, 16, 17, 18, 20, 21, 25, 26, 28, 29], "120": [4, 16, 18, 28, 29], "121": 18, "122": [16, 28, 29], "123": [16, 28, 29], "12371": 4, "125": [4, 18, 21, 37], "128": 4, "13": [16, 18, 20, 21, 25, 26, 28, 29], "130": [16, 18, 28, 29], "135": [16, 28, 29], "136": 18, "14": [16, 17, 18, 20, 21, 24, 25, 28, 29], "140": 18, "14285714285714": 18, "145": [16, 28, 29], "149": 37, "15": [16, 17, 18, 20, 21, 24, 25, 26, 28, 29], "150": [16, 18, 28, 29], "158": 37, "159": [16, 19, 28, 29, 37], "16": [17, 18, 20, 21, 25], "160": 37, "161": 37, "162": 37, "163": 37, "165": [21, 37], "169": 19, "17": [17, 18, 20, 21, 25, 26], "18": [17, 18, 20, 21, 25, 26], "182": 19, "19": [18, 20, 21, 37], "190": 37, "1902": 4, "1921": 21, "195": [16, 28, 29], "1970": 21, "2": [2, 3, 4, 13, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "20": [2, 4, 5, 13, 16, 18, 20, 21, 26, 28, 29, 37], "200": 37, "2025": 21, "205": [16, 28, 29], "21": [17, 18, 20, 21, 26], "22": [2, 20, 21, 28], "223": 37, "224": 4, "229": 37, "23": [17, 20, 26, 37], "23076923076923": 18, "2345": 4, "24": [17, 20], "24t20": 21, "25": [4, 16, 18, 20, 21, 26, 28, 29], "256": 4, "25806451612904": 18, "26": [20, 26], "27": 20, "28": [17, 20, 26], "28571428571429": 18, "29": 20, "3": [2, 4, 13, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "30": [16, 18, 20, 21, 26, 28, 29], "309": [16, 21, 28, 29], "31": 20, "314": [16, 21, 28, 29], "318": [16, 21, 28, 29], "32": 20, "325": 19, "33": [17, 20], "333333333333332": 26, "333333333333336": 18, "34": [20, 21], "35": [16, 17, 18, 20, 24, 26, 28, 29], "35808733791956": 19, "36": [17, 20], "362": 19, "37": 20, "38": 20, "384": 4, "39": [16, 20, 28, 29, 37], "395": [16, 28, 29], "3rd": 4, "4": [2, 4, 13, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "40": [4, 15, 16, 18, 20, 28, 29], "405": [16, 21, 28, 29], "41": 20, "42": [18, 20, 26, 28], "43": [16, 19, 20, 26, 28, 29, 36], "44": [16, 20, 28, 29], "45": [16, 20, 21, 26, 28, 29], "4579": 21, "46": 18, "47": 18, "4732": 21, "48": [16, 18, 28, 29], "49": [16, 28, 29], "495": [16, 28, 29], "5": [2, 4, 13, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "50": [4, 16, 18, 26, 28, 29], "51": [17, 18], "512": 4, "5129": 21, "52": [16, 28, 29], "525": [16, 21, 28, 29], "53": 18, "530": [16, 28, 29], "534": [16, 21, 28, 29], "54": [18, 20, 21], "55": [16, 18, 26, 28, 29], "56": [18, 26], "58": [16, 18, 28, 29], "5811388300841898": 19, "59": [16, 28, 29], "6": [16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "60": [4, 16, 18, 26, 28, 29], "62": [16, 28, 29], "625": [16, 21, 26, 28, 29], "63": [16, 18, 28, 29], "630": [16, 28, 29], "634": [16, 21, 28, 29], "64": [15, 16, 26, 28, 29], "65": [16, 18, 20, 24, 26, 28, 29], "66": [17, 18], "666666666666668": 26, "66666666666667": 18, "666667": 4, "67": [18, 20, 26], "7": [16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "70": [16, 18, 26, 28, 29], "71": [18, 20], "72": 18, "73": 17, "732": 21, "735522831z": 21, "75": [4, 16, 18, 28, 29], "77777777777777": 18, "78": [16, 17, 28, 29], "785714285714285": 18, "78571428571429": 18, "79": [16, 17, 28, 29], "8": [15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30], "80": [4, 16, 18, 26, 28, 29], "81": [17, 18, 26], "82": [16, 28, 29], "83": [16, 18, 28, 29], "833333333333336": 18, "84": [16, 26, 28, 29], "85": [16, 18, 26, 28, 29], "855": 21, "86": 18, "86740bfd3d9831d6b7c1d0e1bf4a21d91598a0ac": [3, 5], "88888888888889": 18, "9": [4, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "90": [4, 16, 18, 26, 28, 29], "91": 18, "92": 17, "94": [17, 24], "95": [17, 18, 26, 37], "96": [17, 18], "972": 21, "98": [18, 37], "99": 18, "A": [1, 2, 3, 4, 5, 6, 10, 12, 13, 17, 19, 23, 26, 36], "AND": [2, 3, 4, 5, 13], "As": [4, 20, 24, 28], "At": 4, "BY": 13, "Be": 2, "By": [2, 15, 18], "For": [1, 2, 4, 12, 13, 15, 18, 20, 21, 23, 24, 25, 29], "INTO": [1, 5], "If": [1, 2, 3, 4, 5, 6, 13, 15, 18, 21, 23, 25, 26, 29, 36], "In": [13, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 37], "It": [1, 2, 3, 4, 5, 13, 16, 17, 18, 20, 23, 30, 35], "Its": 16, "No": 2, "Not": [3, 5], "OR": [3, 4, 5], "On": [2, 28], "One": [2, 3, 18, 26], "That": [3, 5, 30], "The": [1, 2, 4, 5, 7, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 28], "Then": 25, "There": [4, 21, 25], "These": [4, 5, 11, 13, 18, 20, 27], "To": [2, 4, 15, 20, 21, 24, 25, 26, 28, 29, 30, 36], "With": [3, 5, 29, 30], "__add__": [3, 5], "__aiter__": [5, 11], "__and__": [3, 5], "__anext__": [5, 11], "__arrow_c_array__": [1, 30], "__arrow_c_stream__": [1, 2, 30], "__call__": [5, 13], "__datafusion_table_provider__": [1, 36], "__eq__": [3, 5], "__ge__": [3, 5], "__getitem__": [2, 3, 5], "__gt__": [3, 5], "__init__": [5, 13, 25], "__invert__": [3, 5], "__iter__": [5, 11], "__le__": [3, 5], "__lt__": [3, 5], "__mod__": [3, 5], "__mul__": [3, 5], "__ne__": [3, 5], "__next__": [5, 11], "__or__": [3, 5], "__radd__": [3, 5], "__rand__": [3, 5], "__repr__": [2, 3, 5, 10], "__richcmp__": [3, 5], "__rmod__": [3, 5], "__rmul__": [3, 5], "__ror__": [3, 5], "__rsub__": [3, 5], "__rtruediv__": [3, 5], "__str__": 13, "__sub__": [3, 5], "__truediv__": [3, 5], "__version__": 29, "_aggreg": 4, "_intern": [0, 2, 3, 5, 10, 11, 12], "_null_treat": 3, "_order_bi": 3, "_partition_bi": 3, "_r": [5, 13], "_raw_plan": [5, 10], "_repr_html_": 2, "_sum": [5, 13, 25], "_to_pyarrow_typ": [3, 5], "_udaf": [5, 13], "_udf": [5, 13], "_udwf": [5, 13], "_w": 13, "_window_fram": 3, "_window_funct": 4, "a0": 20, "a_siz": 20, "ab": 4, "abc": 6, "abi3": 15, "abil": 28, "abilti": 28, "abl": [5, 10], "about": [12, 16, 26, 27], "abov": [4, 18, 26], "absolut": 4, "abstract": [5, 6, 13, 20, 25], "accept": [1, 3, 5, 30], "access": [20, 28], "access_key_id": 28, "account": 28, "accum": [5, 13], "accumul": [5, 13, 25], "achiev": 16, "aco": 4, "acosh": 4, "across": [13, 20], "act": [5, 10], "activ": 15, "actual": 2, "ad": 2, "add": [1, 2, 3, 4, 15], "add_3": 2, "addit": [2, 3, 4, 5, 12, 13, 15, 25, 28], "addition": 17, "adhoc": 15, "advanc": 28, "advantag": [15, 16], "affect": [13, 18], "after": [2, 4, 13, 25], "ag": 20, "against": [1, 3, 4, 13, 16], "age_col": 20, "age_in_year": 20, "agg": 2, "aggreg": [1, 2, 3, 4, 5, 13, 17, 22], "aggregatefunct": 3, "aggregateudf": [1, 5, 13], "aim": 29, "albert": 20, "algorithm": [2, 4], "alia": [1, 2, 3, 4, 5, 17, 18, 20, 21, 24, 25, 26, 30], "alias": 15, "alic": 23, "all": [0, 1, 2, 4, 5, 13, 15, 17, 18, 21, 23, 24, 25, 26, 28], "alloc": 2, "allow": [1, 5, 13, 15, 16, 18, 20, 21, 25], "alpha": [25, 28], "alreadi": 4, "also": [1, 2, 5, 13, 15, 16, 18, 20, 21, 25, 26, 29, 37], "altern": [24, 32, 35], "alternate_a": 2, "alwai": 13, "amazons3": [9, 28], "amount": [2, 13], "an": [1, 2, 3, 4, 5, 10, 12, 13, 15, 17, 18, 19, 20, 23, 24, 25, 26, 28, 29, 30, 31, 32, 35], "analyt": 26, "analyz": [2, 3, 15], "ani": [1, 2, 3, 4, 5, 6, 8, 10, 18, 23, 25, 26, 28, 30], "anoth": [2, 4], "anti": [2, 22], "apach": [1, 3, 4, 5, 15, 16, 30], "apart": 16, "api": [1, 5, 10, 15, 16, 21, 26, 27, 37], "appear": [3, 4, 5], "append": [4, 25], "appli": 2, "approach": [4, 18, 20], "appropri": [13, 15, 28], "approx_distinct": [4, 18], "approx_median": [4, 18], "approx_percentile_cont": [4, 18], "approx_percentile_cont_with_weight": [4, 18], "approxim": [4, 18], "ar": [1, 2, 3, 4, 5, 10, 11, 13, 15, 17, 18, 20, 22, 24, 25, 26, 28, 29, 30], "arbitrari": [2, 4], "arc": [4, 36], "arg": [1, 2, 4, 5, 12, 13, 15], "argument": [2, 4, 5, 13, 18, 25], "around": [4, 15], "arrai": [1, 2, 3, 4, 5, 13, 18, 22, 25, 28, 30], "array1": 4, "array2": 4, "array_agg": [4, 18], "array_append": [4, 15], "array_cat": [4, 20], "array_concat": [4, 20], "array_dim": 4, "array_distinct": 4, "array_el": [3, 4, 5, 20], "array_empti": [4, 20], "array_except": 4, "array_extract": 4, "array_ha": 4, "array_has_al": 4, "array_has_ani": 4, "array_indexof": 4, "array_intersect": 4, "array_join": 4, "array_length": 4, "array_ndim": 4, "array_pop_back": 4, "array_pop_front": 4, "array_posit": 4, "array_prepend": 4, "array_push_back": 4, "array_push_front": 4, "array_remov": 4, "array_remove_al": 4, "array_remove_n": 4, "array_repeat": [4, 20], "array_replac": 4, "array_replace_al": 4, "array_replace_n": 4, "array_res": 4, "array_slic": 4, "array_sort": 4, "array_to_str": 4, "array_union": 4, "arrow": [1, 2, 3, 4, 5, 15, 16, 19, 28, 29, 33], "arrow_cast": [3, 4, 5, 21], "arrow_datafusion_python_root": 15, "arrow_typeof": 4, "arrowarrayexport": 1, "arrowstreamexport": 1, "articuno": 26, "arxiv": 4, "as_pi": [5, 13, 25], "ascend": [3, 4, 5, 18, 26], "ascii": 4, "asin": 4, "asinh": 4, "assembl": 1, "assign": [3, 4, 5], "assist": 15, "associ": [0, 1, 3, 4, 5], "assum": [2, 4, 15, 28], "assumpt": 16, "async": [5, 11], "atan": 4, "atan2": 4, "atanh": 4, "atk": [16, 28, 29], "attack": [16, 18, 21, 26, 28, 29, 37], "attempt": [1, 2, 5, 13, 15], "auto": 14, "autoapi": 14, "automat": [1, 5], "avail": [2, 18, 22, 25, 27, 36], "availbl": 26, "averag": [4, 13, 26], "avg": [4, 13, 18, 26], "avoid": [1, 5], "avro": [1, 28, 33], "awar": [2, 24], "aws_access_key_id": 28, "aws_secret_access_kei": 28, "b": [2, 4, 19, 20, 25, 28, 30], "back": [3, 16, 30], "bar": 27, "base": [1, 2, 3, 4, 5, 7, 8, 13, 14, 20, 23, 25], "base64": 4, "baseinputsourc": [6, 8], "basi": [5, 13, 25], "basic": [17, 22, 24, 25], "basic_typ": 21, "batch": [1, 2, 5, 10, 11, 13, 19, 25, 28, 30], "batch_arrai": 25, "batch_siz": [1, 5], "becaus": [3, 5, 25], "becca": 20, "beedril": [16, 26, 28, 29], "beedrillmega": [16, 28, 29], "befor": [4, 15, 20, 25], "beforehand": [1, 5], "begin": [1, 3, 4, 5, 13], "beginnig": 4, "being": 2, "below": [17, 18, 28], "best": [1, 5], "beta": 28, "better": 2, "between": [1, 3, 4, 5, 13, 16, 17, 23, 25, 26], "betwen": 26, "bia": [5, 13], "bias_10": [5, 13], "biasednumb": [5, 13], "bin": [4, 15], "binari": [3, 4, 5, 10], "binaryexpr": [3, 5], "bind": [5, 16, 29], "bit": 4, "bit_and": [4, 18], "bit_length": 4, "bit_or": [4, 18], "bit_xor": [4, 18], "bitwis": [4, 20], "blake2": 4, "blake2b": 4, "blake3": 4, "blastois": [16, 28, 29], "blastoisemega": [16, 28, 29], "blob": [3, 5], "blog": 15, "blue": 20, "bob": 23, "bool": [1, 2, 3, 4, 5, 6, 13], "bool_": 25, "bool_and": [4, 18], "bool_or": [4, 18], "boolean": [4, 22], "bootstrap": 15, "both": [2, 4, 15, 20, 23], "bound": [3, 5, 13, 26, 36], "boundari": [13, 26], "bow": 26, "bracket": 20, "bronz": 4, "brotli": 2, "btrim": 4, "bucket_nam": 28, "bug": [16, 18, 26, 28, 29], "build": [3, 4, 5, 16, 17, 18, 26], "build_flag": 15, "build_tabl": [6, 8], "builder": [3, 4, 18, 26], "built": [13, 18, 20, 21, 25], "bulb": 21, "bulbafleur": 21, "bulbasaur": [16, 21, 26, 28, 29], "butterfre": [16, 26, 28, 29], "byte": [1, 4, 5, 10, 12], "c": [1, 2, 13, 15, 16, 19, 28, 30], "cach": 2, "calcul": [2, 4, 13, 25], "call": [0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 17, 18, 20, 25, 30], "callabl": [2, 5, 13], "calll": 20, "can": [1, 2, 3, 4, 5, 6, 8, 10, 13, 15, 17, 18, 20, 24, 25, 26, 27, 28, 29, 30, 36], "can_retir": 20, "cannot": 4, "canonical_nam": [3, 5], "capit": [4, 24], "cardin": [4, 20], "cargo": 15, "carlo": 20, "case": [2, 3, 4, 5, 13, 18, 21, 24, 25, 26, 28], "case_build": 3, "casebuild": [3, 4], "cast": [2, 3, 4, 5, 22], "castabl": [3, 5], "catalog": [1, 5, 14], "caterpi": [16, 26, 28, 29], "caus": [1, 5, 15, 26, 30], "cbrt": 4, "cdatainterfac": 1, "ceil": 4, "certain": 13, "certainli": 15, "chain": 2, "chang": [13, 15, 20], "chansei": 18, "char": 21, "char_length": [4, 21], "charact": 4, "character_length": 4, "charizard": [16, 21, 26, 28, 29], "charizardmega": [16, 21, 26, 28, 29], "charli": 23, "charmand": [16, 21, 26, 28, 29], "charmeleon": [16, 21, 26, 28, 29], "check": [4, 20, 21], "checksum": 4, "children": [5, 10], "chr": 4, "chrono": 4, "ci": 15, "citycab": 23, "class": [7, 25], "classmethod": 2, "claus": [13, 26], "clean": 15, "clefabl": 26, "clefairi": [18, 26], "clone": [15, 36], "close": 26, "co": 4, "coalesc": [4, 21], "code": [4, 21], "codebas": 15, "coeffici": 4, "coerc": 4, "col": [2, 3, 4, 5, 13, 17, 18, 20, 21, 24, 25, 26, 30, 37], "col1": 13, "col_attack": 18, "col_diff": 25, "col_spe": 18, "col_type_1": 18, "col_type_2": 18, "collect": [2, 16, 17, 19, 30, 37], "collect_partit": 2, "color": 20, "column": [1, 2, 3, 4, 5, 13, 17, 18, 19, 21, 22, 23, 25, 26, 37], "column_a": 3, "column_nam": [3, 5], "com": [3, 4, 5, 15], "combin": [2, 4, 20, 23, 24, 25], "come": [28, 34], "command": [1, 5, 15], "common": [3, 4, 5, 6, 8, 18, 21, 23, 26, 30], "commun": 15, "compar": [4, 26], "comparison": [3, 5, 18], "compat": 20, "compil": 20, "complet": [3, 4, 5, 6, 15, 25, 36], "complex": [2, 16, 21], "complic": 2, "compon": 4, "compos": 4, "compress": [1, 2], "compression_level": 2, "comput": [2, 3, 4, 5, 13, 20, 25], "concat": 4, "concat_w": 4, "concaten": [4, 20], "concatenated_arrai": 20, "concept": [1, 2, 3, 5, 20], "concis": 15, "concurr": [1, 5], "condit": 22, "config": [1, 5, 15, 27], "config_intern": [1, 5], "config_opt": [1, 5], "configur": [1, 5, 15], "conflict": 1, "conjunct": 2, "connect": [1, 17], "consecut": 4, "consid": 4, "consist": 4, "consol": 2, "construct": [3, 5, 13, 25], "constructor": [0, 2, 3, 5, 10, 11, 12, 13], "consum": [6, 12, 30], "contain": [2, 4, 5, 10, 13, 14, 15, 18, 20, 23, 25], "content": [15, 22], "context": [4, 5, 10, 12, 14, 27, 30, 37], "continu": 4, "contribut": 15, "control": [1, 5, 18, 26], "conveni": [5, 13, 28], "convers": 15, "convert": [1, 2, 3, 4, 5, 10, 11, 12, 17, 19, 20, 21, 25, 37], "copi": [15, 16, 25, 30], "core": [2, 3, 5], "corr": [4, 18], "correctli": [2, 13], "correl": 4, "correspond": [4, 23], "cosh": 4, "cosin": 4, "cost": 16, "costli": 13, "cot": 4, "cotang": 4, "could": [2, 3, 5], "count": [2, 4, 18, 19], "count_star": 4, "counterpart": 4, "coupl": 26, "covar": 4, "covar_pop": [4, 18], "covar_samp": [4, 18], "covari": 4, "cover": [17, 21, 27], "cpython": 15, "creat": [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 20, 27, 30, 37], "create_datafram": [1, 25, 28], "create_dataframe_from_logical_plan": 1, "creatememoryt": 3, "createview": 3, "credenti": 28, "criteria": [4, 26], "cstring": 36, "csv": [1, 2, 16, 18, 21, 26, 28, 29, 33, 37], "ctx": [1, 4, 5, 10, 12, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37], "cube": 4, "cubon": 18, "cume_dist": [4, 13, 26], "cumul": 4, "curr_valu": 25, "current": [2, 3, 4, 5, 10, 13, 21, 25, 26], "current_d": 4, "current_tim": 4, "custom": [1, 5, 6, 10, 23, 25, 33], "customer_id": 23, "d": [2, 13, 30], "dai": [4, 21], "dant": 20, "dark": 18, "data": [0, 1, 2, 3, 4, 5, 10, 13, 15, 16, 17, 19, 20, 21, 24, 25, 29, 30, 31, 34, 36], "data_typ": 4, "databas": [0, 5, 20], "dataflow": [5, 10], "datafram": [1, 3, 4, 5, 10, 11, 13, 14, 16, 18, 19, 20, 21, 23, 24, 25, 26, 29, 30, 37], "datafrm": 2, "datafus": [14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 37], "datafusion_df": 28, "datafusion_table_provid": 36, "dataset": [1, 21, 26, 28, 29], "datasourc": 6, "datastructur": [3, 5], "datatyp": [2, 3, 5, 13], "datatypemap": [3, 5], "date": [4, 21], "date32": 4, "date_bin": 4, "date_part": [4, 21], "date_trunc": 4, "datepart": 4, "datetrunc": 4, "db": [0, 5], "ddl": [1, 5], "decid": 13, "decim": 4, "decimal_plac": 4, "decod": 4, "def": [2, 5, 13, 16, 25, 28, 29], "default": [1, 2, 3, 4, 5, 8, 13, 15, 25, 26, 27], "default_valu": 4, "defens": [16, 21, 28, 29, 37], "defin": [1, 3, 5, 13, 15, 21, 22, 26], "definit": [1, 3, 5, 13, 17, 25], "degre": 4, "delet": [1, 5], "delimit": [1, 4], "delta_t": 28, "deltalak": 28, "deltat": 28, "demonstr": [1, 25, 26, 29, 30], "dens": 4, "dense_rank": [4, 13, 26], "depend": [4, 18, 25, 26], "deprec": [2, 4, 26], "deregister_t": 1, "descend": 4, "describ": [2, 4, 17, 18, 19, 28], "descript": [4, 5, 13], "deseri": 12, "deserialize_byt": 12, "design": 22, "desir": 4, "detail": [2, 3, 4, 5, 13, 15, 19], "determin": [2, 3, 4, 5, 25], "dev": 15, "deviat": 4, "df": [1, 2, 3, 4, 16, 17, 18, 19, 20, 21, 24, 25, 26, 28, 29, 30, 31, 32, 34, 35, 37], "dfschema": 5, "dict": [1, 2, 5], "dictionari": [1, 2, 20, 28], "differ": [2, 3, 4, 5, 10, 13, 21, 25, 28], "digest": 4, "dimens": 4, "dimension": 2, "directli": [3, 4, 12, 28, 30], "directori": [1, 5, 15], "disabl": [1, 5], "discard": 2, "disk": [1, 5, 8], "displai": [5, 10, 17, 19, 29], "display_graphviz": [5, 10], "display_ind": [5, 10], "display_indent_schema": [5, 10], "display_nam": [3, 5], "distanc": 4, "distinct": [2, 3, 4, 5], "distribut": 4, "divid": 4, "divis": [3, 4, 5], "divisor": 4, "dml": [1, 5], "do": [1, 5, 13, 15, 17, 18, 25, 26, 28, 36], "doc": [1, 4], "docstr": 15, "document": [1, 2, 3, 4, 5, 14, 15, 17, 27, 29], "doe": [2, 4, 13, 15, 26, 28], "doesn": 13, "dominant_typ": 21, "done": [2, 18, 25], "dot": [5, 10], "doubl": [24, 30], "down": [26, 28], "download": [24, 29], "dragon": [16, 18, 21, 26, 28, 29], "dragonair": 26, "dratini": 26, "drop": [1, 2, 5], "droptabl": 3, "dtype": 2, "due": [1, 15], "duplic": [2, 4, 23], "dure": [5, 13, 15], "dynamic_lookup": 15, "e": [2, 4, 13, 25, 36], "each": [2, 3, 4, 5, 13, 15, 18, 26], "easi": 34, "easier": [21, 29], "easili": [3, 5], "east": 28, "effect": [24, 29], "either": [1, 2, 3, 5, 13, 18], "electr": [18, 26], "element": [2, 3, 4, 5, 20], "ellipsi": [2, 5, 13], "els": 25, "else_expr": 3, "employe": 20, "empti": [1, 4, 5, 10, 18, 20], "empty_t": 1, "emptyrel": 3, "enabl": [1, 5], "enable_url_t": 1, "encod": [4, 12], "encourag": 15, "end": [0, 2, 3, 4, 5, 10, 11, 13, 26], "end_bound": [3, 5], "ends_with": 4, "engin": [1, 5, 16, 17], "enough": 13, "ensur": 6, "entir": [13, 18, 25, 26], "entri": [2, 4, 18, 26], "enum": [2, 3, 5, 13], "enumer": 13, "enviorn": 15, "environ": [15, 16], "equal": [2, 3, 4, 5], "equival": [4, 13, 18, 26], "error": [1, 3, 5, 15], "especi": 15, "essenti": [5, 11, 19], "etc": [1, 3, 5, 8, 13], "eval_rang": 13, "evalu": [2, 3, 4, 5, 13, 17, 18, 20, 25, 26], "evaluate_al": [5, 13, 25], "evaluate_all_with_rank": [13, 25], "even": [1, 5, 23], "everi": [4, 5, 13], "everyth": 8, "ex": [3, 5, 24], "exactli": [2, 26], "examin": [3, 5], "exampl": [1, 2, 3, 4, 5, 10, 13, 17, 18, 20, 21, 23, 25, 26, 28, 29, 36], "except": [2, 3, 4, 5, 20], "except_al": 2, "exclud": [2, 23], "execut": [1, 2, 5, 13, 15, 16, 17, 27, 30], "execute_stream": [2, 5, 11], "execute_stream_partit": 2, "execution_plan": 2, "executionplan": [1, 2, 5, 10], "exeggcut": 18, "exist": [1, 2, 3, 4, 5], "exp": 4, "exp_smooth": 25, "expand": 2, "expect": [3, 4, 5, 13, 20, 25, 26], "expens": 2, "experi": 15, "explain": [2, 3], "explan": [2, 17], "explicit": 27, "expon": 4, "exponenti": 4, "exponentialsmooth": 25, "export": [1, 2, 15, 33], "expos": [1, 28, 36], "expr": [1, 2, 4, 5, 13, 14, 26], "expr1": 4, "expr2": 4, "express": [2, 3, 4, 5, 13, 18, 21, 22, 25, 26], "exprfuncbuild": [3, 5], "extend": [4, 6], "extens": [1, 3], "extract": [3, 4, 5, 21], "extraenv": 15, "f": [3, 17, 18, 20, 21, 26, 28], "fact": 25, "factori": [4, 5, 13], "fail": 15, "fair": [1, 5], "fairi": [18, 26], "fals": [2, 3, 4, 5, 13, 15, 16, 20, 21, 24, 25, 27, 28, 29, 36], "familiar": 15, "far": 2, "faster": [4, 13], "fearow": 26, "featur": [4, 15, 28, 30], "fetch": 15, "few": [15, 17], "ffi": 28, "ffi_tableprovid": [1, 36], "field": [2, 4, 21], "fight": [18, 26], "file": [1, 2, 5, 8, 12, 15, 16, 17, 24, 29, 31, 32, 34, 35], "file_compression_typ": 1, "file_extens": 1, "file_partition_col": 1, "file_sort_ord": 1, "filenam": 15, "fill": [3, 4, 5, 26], "fill_nan": [3, 5], "fill_nul": [3, 5], "filter": [2, 3, 4, 5, 10, 17, 20, 26, 28], "final": [17, 25], "find": [4, 15, 17, 18, 26], "find_in_set": 4, "finish": [3, 18], "fire": [16, 18, 21, 28, 29], "first": [1, 2, 3, 4, 5, 15, 17, 18, 20, 26, 29], "first_1": 18, "first_2": 18, "first_arrai": 4, "first_valu": [4, 13, 18], "fix": 13, "flag": [4, 13, 15], "flatten": 4, "fleur": 21, "float": [3, 4, 5, 13, 25], "float64": [5, 13, 21, 25], "floor": 4, "flow": [5, 10], "flower": 21, "fly": [16, 18, 26, 28, 29], "fn": 36, "folder": 36, "follow": [1, 2, 3, 4, 5, 13, 15, 17, 18, 20, 21, 23, 25, 26, 28, 29], "foo": 27, "fora": [3, 5], "form": [2, 4, 5, 10, 18], "format": [1, 2, 4, 5, 10, 28, 31, 34], "format_argu": 4, "formatt": 4, "forth": 30, "found": [4, 15, 26, 36], "four": 25, "frame": [3, 4, 5, 13, 19, 25], "frame_bound": 3, "framework": 15, "free": [1, 5], "frequent": 15, "from": [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37], "from_arrai": [25, 28], "from_arrow": [1, 28, 30], "from_arrow_t": 1, "from_panda": 1, "from_polar": [1, 28], "from_proto": [5, 10], "from_pydict": [1, 19, 20, 23, 25, 28, 30], "from_pylist": [1, 23, 28], "from_str": 2, "from_substrait_plan": 12, "from_unixtim": 4, "from_val": 4, "fuction": 13, "full": [2, 4, 22, 37], "fulli": 15, "func": [2, 5, 13], "function": [1, 2, 3, 9, 11, 13, 14, 17, 19, 22, 28, 35], "function_to_impl": [13, 25], "futher": 4, "futur": [13, 26], "g": [4, 13], "gamma": 28, "gcd": 4, "gener": [3, 5, 10, 11, 12, 13, 14, 15, 16, 28, 29], "geodud": 26, "get": [2, 5, 10, 13, 15, 20, 28], "get_default_level": 2, "get_frame_unit": [3, 5], "get_lower_bound": [3, 5], "get_offset": 3, "get_rang": 13, "get_upper_bound": [3, 5], "getenv": 28, "ghost": 18, "gil": 16, "git": 15, "github": [3, 4, 5, 15], "give": [4, 18, 24, 29], "given": [0, 1, 3, 4, 5, 13, 25], "go": [19, 21], "gold": 4, "good": 15, "googlecloud": [9, 28], "graph": [5, 10], "graphic": [5, 10], "graphviz": [5, 10], "grass": [16, 18, 21, 26, 28, 29], "great": 15, "greater": [3, 4, 5], "greatest": 4, "greatli": 25, "greedi": [1, 5], "green": 20, "grimer": 26, "ground": 18, "group": [1, 2, 3, 4, 5, 13, 17, 18, 19, 26], "group_bi": [2, 18], "groupingset": 3, "guid": [1, 22, 27, 29], "gzip": 2, "ha": [1, 2, 4, 5, 13, 20, 24, 28], "handl": [11, 18, 26], "happen": [15, 30], "has_head": 1, "hash": [2, 4], "haskel": 19, "have": [1, 2, 4, 5, 12, 13, 15, 16, 18, 23, 25, 26, 36], "head": 2, "header": [1, 2], "healthi": 15, "heavy_red_unit": 20, "help": [1, 15, 20, 21], "helper": [5, 13], "henc": 13, "here": [4, 21, 24, 26, 29, 30, 37], "hex": 4, "hexadecim": 4, "high": [3, 5], "higher": [2, 3, 5], "highli": 25, "highlight": 29, "hint": [1, 15], "homebrew": 15, "host": 1, "how": [1, 2, 3, 4, 5, 13, 18, 19, 22, 23, 25, 26, 27, 29], "howev": [13, 25], "hp": [16, 28, 29], "html": [1, 4], "http": [1, 3, 4, 5, 9, 10, 12, 28], "hyperbol": 4, "i": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13, 15, 16, 17, 18, 20, 21, 23, 25, 26, 28, 29, 30, 31, 32, 34, 35], "ic": [18, 26], "id": [1, 15, 23], "ident": 28, "identifi": [1, 3, 5, 24], "idx": [13, 25], "ignor": [4, 15, 18, 26], "ignore_nul": [18, 26], "ilik": 3, "immut": [5, 13, 25], "impl": 36, "implement": [1, 2, 13, 15, 25, 28, 30, 36], "import": [1, 3, 5, 13, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37], "improv": [1, 5, 13, 25], "in_list": [4, 21], "includ": [1, 2, 3, 5, 18, 21, 23], "include_rank": [13, 25], "inclus": [1, 3, 4, 5], "incorpor": 25, "increas": [1, 5, 18], "increment": [13, 25], "indent": [5, 10], "independ": [4, 26], "index": [3, 4, 5, 13, 20], "indic": [3, 4, 20], "individu": [20, 26], "infer": 1, "inform": [1, 2, 3, 5, 6, 12, 25], "information_schema": [1, 5], "init": 15, "initcap": 4, "initi": 4, "inlin": 13, "inlist": 3, "inner": [2, 22], "input": [1, 2, 4, 5, 10, 13, 14, 20, 25], "input_fil": 8, "input_item": [6, 8], "input_typ": [5, 13], "inputsourc": 6, "insert": [1, 5], "instanc": [1, 4, 5, 13, 17], "instanti": [5, 13], "instead": [3, 4, 5], "insubqueri": 3, "int": [1, 2, 3, 4, 5, 10, 13, 25], "int32": 21, "int64": [5, 13, 19, 21, 25], "integ": [3, 4, 5, 20], "integr": [29, 36], "interact": 20, "intercept": 4, "interchang": 34, "interest": 20, "interfac": [1, 4, 16, 17, 25, 26, 28, 30, 36], "intermedi": [5, 13], "intern": 25, "intersect": [2, 4], "interv": 4, "intro": 15, "introduc": [15, 17], "introduct": 17, "invers": 4, "invoc": 13, "io": [1, 12], "is_caus": 13, "is_correct_input": [6, 8], "is_current_row": 3, "is_empti": 20, "is_follow": 3, "is_not_nul": [3, 5, 18], "is_nul": [3, 5, 25], "is_null_arr": 25, "is_preced": 3, "is_unbound": 3, "isfals": 3, "isnan": 4, "isnotfals": 3, "isnotnul": [3, 5], "isnottru": 3, "isnotunknown": 3, "isnul": 3, "issu": [4, 15, 26], "istru": 3, "isunknown": 3, "iszero": 4, "iter": [2, 5, 11], "iterat": 2, "its": [2, 4, 5, 10, 13, 16, 26], "itself": 20, "ivi": 21, "ivyfleur": 21, "ivysaur": [16, 21, 26, 28, 29], "java": 19, "javascript": 34, "jigglypuff": 26, "join": [1, 2, 3, 5, 10, 22], "join_kei": [2, 23], "join_on": 2, "joinconstraint": 3, "jointyp": 3, "json": [1, 2, 28, 33], "jupyt": 29, "jupyterlab": 29, "jynx": [18, 26], "kakuna": [16, 26, 28, 29], "keep": 15, "kei": [1, 2, 3, 5, 17, 20, 23], "kind": [0, 5, 15], "know": [1, 5], "known": 20, "kwarg": [6, 8], "kwd": [2, 13], "l179": [3, 5], "lab": 29, "label": 2, "lack": 16, "lag": [4, 13, 26], "lambda": [5, 13], "languag": [1, 5, 10], "larg": [2, 21], "large_trip_dist": 24, "last": [4, 26], "last_valu": [4, 18, 26], "last_with_nul": 26, "last_wo_nul": 26, "later": [28, 29, 36], "latest": 4, "latter": 1, "lazi": 17, "lcm": 4, "lead": [4, 13, 25, 26, 28], "leaf": [5, 10], "leak": 16, "learn": [19, 26], "least": [4, 13, 18, 23, 25], "leav": [5, 10], "left": [2, 4, 21, 22], "left_on": [2, 23], "leftmost": 4, "legendari": [16, 28, 29], "len": [13, 21], "length": [4, 5, 13], "less": [3, 4, 5], "let": [27, 36], "letter": [4, 24], "level": [2, 4], "levenshtein": 4, "lib": 15, "lib_dir": 15, "lib_nam": 15, "librari": [5, 6, 16, 29, 30, 36], "lightweight": 34, "like": [1, 3, 4, 5, 6, 15, 16, 21], "limit": [2, 3, 4, 15, 17, 19, 21], "line": [1, 2, 5, 10], "linear": [4, 18], "link": [15, 26], "lint": 15, "linter": 15, "list": [0, 1, 2, 3, 4, 5, 10, 13, 18, 25, 26, 28], "list_append": [4, 15], "list_cat": 4, "list_concat": 4, "list_dim": 4, "list_distinct": 4, "list_el": 4, "list_except": 4, "list_extract": 4, "list_indexof": 4, "list_intersect": 4, "list_join": 4, "list_length": 4, "list_ndim": 4, "list_posit": 4, "list_prepend": 4, "list_push_back": 4, "list_push_front": 4, "list_remov": 4, "list_remove_al": 4, "list_remove_n": 4, "list_repeat": 4, "list_replac": 4, "list_replace_al": 4, "list_replace_n": 4, "list_res": 4, "list_slic": 4, "list_sort": 4, "list_to_str": 4, "list_union": 4, "listingt": 1, "lit": [2, 3, 4, 5, 15, 17, 18, 20, 24, 26, 30], "liter": [2, 3, 4, 5, 21, 22, 24], "ll": [21, 23, 26], "ln": 4, "local": [1, 15], "localfilesystem": [9, 28], "locat": [1, 7, 14], "locationinputplugin": [7, 8], "lock": 16, "log": [4, 21], "log10": 4, "log2": 4, "logarithm": 4, "logic": [1, 2, 3, 5, 10, 12, 17, 18, 24], "logical_plan": [2, 12], "logicalplan": [1, 2, 3, 5, 10, 12], "lonely_trip": 24, "long_tim": 20, "longer": 4, "look": [15, 27], "low": [3, 5, 18], "low_passenger_count": 24, "lower": [3, 4, 5, 13, 21, 24], "lowercas": [2, 4], "lowest": 18, "lpad": 4, "ltrim": 4, "lz4": 2, "lz4_raw": 2, "lzo": 2, "m": 15, "mac": 15, "machin": 15, "machop": 26, "made": 24, "magikarp": 18, "magnemit": 26, "mai": [1, 2, 4, 5, 13, 25, 28, 29, 30], "main": [1, 17], "maintain": [1, 2, 17], "major": [15, 16], "make": [4, 15, 16, 21], "make_arrai": 4, "make_d": 4, "make_list": 4, "manag": [1, 5, 15], "mani": [18, 25], "manipul": [1, 5, 21], "mankei": 26, "manual": 15, "map": 2, "map_err": 36, "match": [2, 3, 4, 23], "math": 18, "mathemat": [22, 24], "maturin": 15, "max": [4, 18, 19], "maximum": [1, 4], "md5": 4, "mean": [4, 15, 19], "meaning": [4, 24], "meant": [5, 10], "medal": 4, "median": [4, 18, 19], "member": 15, "memoiz": 13, "memori": [1, 2, 5, 10, 13, 16], "mention": 20, "merg": [5, 13, 23, 25], "metadata": 1, "metapod": [16, 18, 26, 28, 29], "method": [1, 2, 4, 5, 13, 17, 19, 23, 25], "metric": 2, "metrorid": 23, "microsecond": 4, "microsoftazur": [9, 28], "might": [13, 18], "millisecond": 4, "min": [4, 18, 19], "minimum": [1, 4, 5], "model": 2, "modifi": [2, 17], "modify_df": 2, "modul": 26, "modulo": [3, 5], "moment": 2, "month": [4, 21], "more": [1, 2, 3, 5, 12, 15, 18, 21, 25, 27], "most": [13, 20, 26, 29], "mostli": 15, "much": [13, 15], "multi": 16, "multipl": [1, 2, 3, 4, 5, 13, 18, 21, 23, 25, 26, 28], "must": [1, 2, 3, 4, 5, 13, 18, 20, 24, 25, 26, 30, 36], "my_delta_t": 28, "my_tabl": 36, "my_udaf": 25, "myaccumul": 25, "mytableprovid": 36, "myusernam": 15, "n": [2, 4], "name": [0, 1, 2, 3, 4, 5, 13, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 36], "name_pair": 4, "named_expr": 2, "named_struct": [4, 15], "nan": [3, 4, 5], "nanosecond": 4, "nanvl": 4, "natur": 4, "nearest": 4, "need": [1, 2, 3, 5, 13, 15, 25, 26, 29], "neg": [3, 4], "negat": [3, 4, 5, 21], "new": [1, 2, 3, 4, 5, 13, 15, 20, 22, 36, 37], "new_bound": 36, "new_err": 36, "new_nam": 2, "next": [4, 5, 11], "node": [5, 10], "non": [2, 3, 4, 18, 26], "none": [1, 2, 3, 4, 5, 12, 13, 25, 26, 28], "normal": [18, 26], "not_red_unit": 20, "notat": [20, 34], "note": [1, 2, 3, 5, 17, 20, 30], "notebook": 29, "now": [4, 13, 21], "nr": 19, "nth_valu": [4, 13, 18], "ntile": [4, 26], "null": [2, 3, 4, 5, 19, 23, 25, 30], "null_count": 19, "null_first": 4, "null_treat": [3, 4, 5, 18, 26], "nullabl": 2, "nullif": [4, 21], "nulls_first": [3, 4, 5], "nulltreat": [3, 4, 5, 18, 26], "num": [2, 4], "num_centroid": 4, "num_el": 20, "num_row": [5, 13, 25], "number": [1, 2, 4, 5, 10, 13, 15, 20, 21, 26], "numebr": 4, "numer": [2, 4], "nvl": 4, "o": 28, "object": [1, 2, 3, 5, 9, 20, 25, 27, 30, 34], "object_stor": [5, 14, 28], "objectstor": 1, "obtain": [6, 16], "occur": [5, 13, 30], "occurr": 4, "octet_length": 4, "oddish": 26, "offend": 15, "offer": [21, 37], "offset": [2, 3], "often": 26, "old": 2, "old_nam": 2, "older": 28, "olymp": 4, "omit": 23, "on_expr": 2, "onc": [4, 6, 13, 36], "one": [2, 3, 4, 5, 13, 18, 23, 25, 26, 28], "onli": [1, 2, 3, 4, 5, 13, 18, 23, 25, 26], "onlin": [1, 2, 3, 4, 5, 27], "op": [2, 3, 5], "oper": [1, 2, 3, 4, 5, 10, 11, 13, 16, 18, 20, 24, 25, 26, 28, 30], "operand": [3, 5], "opt": 15, "optim": [2, 13], "optimized_logical_plan": 2, "option": [1, 4, 5, 15, 18, 21, 27, 28, 29], "options_intern": [1, 5], "order": [1, 2, 3, 4, 5, 6, 13, 20], "order_bi": [3, 4, 5, 18, 26], "org": [1, 4, 5, 10], "origin": [2, 4, 25], "other": [1, 2, 3, 4, 5, 13, 15, 16, 20, 22], "otherwis": [3, 4], "our": [15, 18, 29], "out": [2, 4, 18, 19, 20, 21, 23, 24, 26, 29, 30, 37], "output": [2, 3, 5, 10, 13, 15, 18, 20, 25], "over": [2, 3, 5, 13, 16, 18, 26], "overhead": 13, "overlai": 4, "overlap": 4, "overrid": 25, "overridden": 18, "own": [6, 25], "pa": [3, 5, 13, 25, 28, 30], "packag": [4, 7, 15], "pad": 4, "page": 14, "pair": 4, "panda": [1, 2, 17, 19, 28, 37], "pandas_df": 28, "para": 26, "parallel": [1, 5], "paramet": [1, 2, 3, 4, 5, 12, 13, 15, 20, 22, 23], "parasect": 26, "parquet": [1, 2, 5, 16, 17, 24, 27, 28, 33], "parquet_prun": 1, "part": [4, 21], "particular": 4, "partit": [1, 2, 3, 4, 5, 10, 13, 18], "partition_bi": [3, 4, 5, 26], "partition_count": [5, 10], "pass": [2, 5, 13, 15, 19, 20, 25, 27, 28], "passenger_count": 24, "path": [1, 2, 5, 12, 15, 28], "path_to_t": 28, "pathlib": [1, 2, 5, 12], "pattern": 4, "pc": [5, 13], "pcre": 4, "pd": 28, "peopl": 4, "per": [2, 13, 26], "percent": 4, "percent_rank": [4, 13, 26], "percentag": 4, "percentil": 4, "perform": [1, 2, 3, 4, 5, 13, 15, 17, 18, 21, 25, 26, 28], "person": 15, "physic": [2, 5, 10, 17], "pi": 4, "pick": 4, "pidgei": 26, "pidgeot": 26, "pidgeotmega": 26, "pidgeotto": 26, "pinsir": 26, "pip": [16, 29], "pl": 28, "place": [4, 15, 29], "placehold": 3, "plan": [1, 2, 3, 5, 12, 13, 14, 16, 17], "plan_intern": 12, "planner": [5, 10], "pleas": 24, "plu": 4, "plugin": [6, 8], "point": [4, 15], "pointer_width": 15, "poison": [16, 18, 26, 28, 29], "pokemon": [16, 17, 18, 21, 26, 28, 29, 37], "polar": [1, 2, 28], "polars_df": 28, "pool": [1, 5], "popul": 4, "popular": [21, 28], "posit": [4, 18], "possibl": [3, 5, 13, 25, 26], "post": 15, "potenti": [2, 5, 10], "pow": [4, 21], "power": [4, 21, 25, 26], "pr": 15, "prece": 4, "preced": [3, 5, 13], "preceed": 26, "precis": 4, "predic": [1, 2, 5], "prefer": 15, "prefix": 4, "prepend": 4, "presenc": 21, "present": 23, "preserve_nul": 2, "prevent": 15, "previou": [4, 25, 26], "previous": 1, "primari": [7, 20], "print": [2, 5, 10, 27], "printabl": [5, 10], "prior": [2, 28], "probabl": 18, "process": [13, 18, 26], "produc": [4, 5, 10, 12, 13, 26], "product": 4, "programmat": [5, 10], "project": [2, 3, 5, 10, 15, 30], "properti": [0, 3, 5, 10], "proto_byt": 12, "protobuf": [5, 10], "protocol": 1, "provid": [0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 17, 18, 21, 25, 26, 29, 33], "prune": [1, 5], "psychic": 18, "public": [0, 5], "pull": 15, "pure": 15, "push": [15, 28], "pushdown_filt": 27, "put": 24, "py": [1, 2, 5, 12, 36], "pyarrow": [0, 1, 2, 3, 5, 11, 13, 19, 25, 28, 30, 37], "pycapsul": [1, 2, 28, 30, 36], "pycapsuleinterfac": 1, "pymethod": 36, "pyo3": 15, "pyo3_build_config": 15, "pyo3_config_fil": 15, "pyo3_print_config": 15, "pyobject": [3, 5], "pypi": 29, "pyproject": 15, "pyresult": 36, "pyruntimeerror": 36, "pyspark": 16, "pytest": 15, "python": [1, 3, 5, 13, 19, 20, 25, 28, 29, 30, 36], "python3": 15, "python_valu": [3, 5], "pythontyp": [3, 5], "queri": [1, 5, 10, 12, 13, 16, 17, 21], "quick": 19, "quit": 35, "quot": 24, "r": [2, 3, 4, 5, 15], "radian": 4, "rais": [2, 4], "random": [4, 13, 19, 20], "rang": [1, 2, 3, 4, 5, 13, 19, 20, 21, 25, 26, 27], "rank": [4, 13, 25, 26], "ranks_in_partit": 13, "rather": [4, 12], "ratic": 26, "ratio": 4, "rattata": 26, "raw": 3, "raw_sort": 3, "rb": [5, 11], "read": [1, 8, 12, 27, 28, 29, 31, 32, 34, 35, 37], "read_avro": [1, 31], "read_csv": [1, 16, 17, 18, 26, 28, 29, 32], "read_json": [1, 34], "read_parquet": [1, 17, 24, 35], "read_tabl": 1, "reader": [1, 5, 30], "realiti": 18, "reason": 15, "rebuild": 15, "receiv": [4, 13], "recent": [26, 28], "recommend": [2, 15, 25], "record": [1, 2, 5, 10, 11, 24, 25, 28, 30, 31], "record_batch": [1, 2, 5, 14], "record_batch_stream": [5, 11], "recordbatch": [1, 2, 5, 11, 25, 28], "recordbatchstream": [1, 2, 5, 11], "red": 20, "red_or_green_unit": 20, "red_unit": 20, "reduc": [15, 25], "ref": 4, "refer": [2, 4, 17, 21, 37], "referenc": [1, 17], "regardless": 13, "regener": 15, "regex": 4, "regexp_lik": 4, "regexp_match": [4, 21], "regexp_replac": [4, 21], "region": 28, "regist": [1, 6, 17, 28, 36, 37], "register_avro": 1, "register_csv": [1, 21, 32, 37], "register_dataset": [1, 28], "register_json": 1, "register_listing_t": 1, "register_object_stor": [1, 28], "register_parquet": [1, 28, 35], "register_record_batch": 1, "register_t": 1, "register_table_provid": [1, 28, 36], "register_udaf": 1, "register_udf": 1, "register_udwf": 1, "regr_avgi": [4, 18], "regr_avgx": [4, 18], "regr_count": [4, 18], "regr_intercept": [4, 18], "regr_r2": [4, 18], "regr_slop": [4, 18], "regr_sxi": 4, "regr_sxx": [4, 18], "regr_syi": [4, 18], "regress": [4, 18], "regular": [4, 21], "rel": [4, 26], "relat": [5, 10, 23], "releas": 26, "relev": 15, "reli": 15, "remain": 4, "remot": 8, "remov": [1, 2, 4, 18], "renam": 2, "renamed_ag": 20, "repartit": [1, 2, 3, 5], "repartition_by_hash": 2, "repeat": [4, 20], "repeated_arrai": 20, "replac": 4, "repo": 15, "report": [2, 15], "repositori": 28, "repres": [1, 2, 3, 4, 5, 10, 11, 12, 17, 20, 25], "represent": [1, 2, 3, 4, 5, 10], "request": [2, 15], "requested_schema": [1, 2], "requir": [2, 3, 13, 21, 28], "reserv": [1, 5], "resolv": [15, 26], "respect": [2, 4, 18, 28], "respect_nul": [4, 18, 26], "result": [1, 2, 4, 5, 10, 11, 13, 16, 17, 18, 20, 23, 24, 25, 26], "retriev": [1, 3, 5, 23], "return": [0, 1, 2, 3, 5, 6, 8, 10, 12, 13, 20, 23, 25, 30], "return_typ": [5, 13], "reusabl": 3, "revers": 4, "review": 15, "rex": [3, 5], "rex_call_oper": [3, 5], "rex_call_operand": [3, 5], "rex_typ": [3, 5], "rextyp": [3, 5], "rfc3339": 4, "rh": [3, 5], "rhyhorn": 26, "right": [2, 4, 23], "right_on": [2, 23], "ritchi": 26, "robin": 2, "rock": 18, "roll": 26, "root": [4, 5, 10], "round": [2, 4, 17], "row": [1, 2, 3, 4, 5, 13, 17, 18, 19, 20, 23, 25, 26, 37], "row_numb": [4, 13, 26], "rpad": 4, "rtrim": 4, "rubi": 19, "run": [1, 2, 5, 16, 29], "runnabl": 15, "runtim": [1, 5, 13, 27], "runtimeconfig": 1, "runtimeenvbuild": [1, 5, 27], "rust": [4, 16, 27, 28, 36], "rustflag": 15, "s3": 28, "safeti": 16, "same": [2, 4, 5, 13, 17, 25], "sampl": [4, 19], "saur": 21, "save": 13, "scalar": [3, 5, 13, 20, 22], "scalarsubqueri": 3, "scalarudf": [1, 5, 13], "scalarvalu": 13, "scalarvari": 3, "scan": [1, 5, 10], "schema": [0, 1, 2, 3, 5, 10, 19], "schema_infer_max_record": 1, "schema_nam": [3, 5], "scheme": 2, "scienc": 21, "score": 4, "search": 4, "second": [4, 17, 18, 20], "second_arrai": 4, "secret_access_kei": 28, "section": [17, 19, 22, 23, 26, 28], "see": [1, 2, 3, 4, 5, 11, 12, 13, 15, 18, 20, 25, 26], "seen": [13, 18], "select": [1, 2, 3, 4, 5, 13, 17, 20, 21, 22, 25, 26, 30, 37], "select_column": 2, "self": [5, 11, 13, 25, 36], "semi": [2, 22], "sensit": 2, "separ": 4, "seper": 4, "sequenc": 2, "serd": 12, "seri": 20, "serial": [12, 31], "serialize_byt": 12, "serialize_to_plan": 12, "session": [1, 5, 27, 29], "session_id": 1, "sessionconfig": [1, 5, 27], "sessioncontext": [1, 2, 4, 5, 6, 10, 12, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37], "set": [0, 1, 3, 4, 5, 13, 15, 17, 22, 25, 27], "sever": 24, "sha": 4, "sha224": 4, "sha256": 4, "sha384": 4, "sha512": 4, "share": [15, 17, 20, 30], "shift_offet": 4, "shift_offset": 4, "shot": 13, "should": [1, 3, 4, 5, 6, 10, 12, 15, 18, 23, 25, 26], "show": [2, 16, 17, 19, 20, 23, 25, 26, 28, 29, 36], "side": 4, "sign": 4, "signific": 28, "significantli": 4, "signum": 4, "silver": 4, "similar": [4, 17, 20, 26], "similarto": 3, "simpl": [4, 15, 24, 34, 35], "simplest": [13, 25], "simpli": [2, 5, 13, 15], "simplifi": 26, "sin": 4, "sinc": [2, 18, 25, 28], "sine": 4, "singl": [1, 2, 3, 4, 5, 10, 13, 18, 20, 25, 26], "sinh": 4, "size": [1, 2, 4, 5, 20], "skip": [1, 2, 5, 13], "skip_metadata": 1, "slice": 4, "slope": 4, "slow": 2, "slower": 2, "slowest": 25, "slowpok": 26, "small": 28, "smallest": [13, 18], "smooth_a": 25, "snappi": 2, "snorlax": 26, "so": [1, 2, 5, 15, 24, 25, 26, 30, 36], "softwar": [5, 10], "some": [4, 13, 15, 18, 21, 26, 29, 36], "sometim": [1, 5, 28], "soon": [4, 28], "sort": [1, 2, 3, 4, 5, 18, 26], "sortexpr": [1, 2, 3, 4, 5], "sourc": [1, 4, 6, 7, 8, 15, 17, 29, 30, 36], "sp": [16, 28, 29], "space": 4, "sparingli": 25, "spearow": 26, "special": 13, "specif": [3, 5, 10, 19, 20, 26], "specifi": [1, 2, 3, 4, 5, 20, 23, 25, 26], "speed": [2, 16, 18, 26, 28, 29], "sphinx": 14, "spill": [1, 5], "spillabl": [1, 5], "split": [4, 25], "split_part": 4, "sql": [1, 5, 10, 12, 16, 17, 20, 24], "sql_with_opt": 1, "sqloption": [1, 5], "sqltabl": [6, 8], "sqltype": [3, 5], "sqrt": 4, "squar": 4, "squi": 21, "squirtl": [16, 21, 26, 28, 29], "src": [3, 5, 15], "stabl": [13, 25], "standard": 4, "start": [3, 4, 5, 13, 20, 24, 26, 29], "start_ag": 20, "start_bound": [3, 5], "started_young": 20, "starts_with": 4, "state": [1, 5, 13, 17, 25], "state_typ": [5, 13], "statement": [1, 3, 4, 5, 17, 37], "static": [3, 5, 10, 12, 13], "statist": [2, 18, 19], "std": 19, "stddev": [4, 18], "stddev_pop": [4, 18], "stddev_samp": 4, "steel": 18, "step": 4, "still": [25, 28], "stop": 4, "storag": 28, "store": [1, 3, 5, 9], "str": [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 13], "str_lit": 21, "straightforward": [31, 32], "stream": [1, 2, 5, 11], "strftime": 4, "stride": 4, "string": [2, 3, 4, 5, 10, 13, 15, 18, 19, 20, 22], "string1": 4, "string2": 4, "string_agg": [4, 18], "string_list": 4, "string_liter": [3, 5, 21], "string_view": [3, 5, 30], "strong": 16, "strpo": 4, "struct": [1, 3, 4, 5, 22, 30], "structarrai": 30, "structur": [5, 10], "style": 20, "sub": [3, 5], "subfield": [3, 4, 5], "submit": 15, "submodul": 15, "subqueri": 3, "subqueryalia": 3, "subset": 24, "substr": 4, "substr_index": 4, "substrait": [5, 14], "subtract": [3, 5], "suffici": [1, 5], "suffix": 4, "suggest": 28, "sum": [4, 5, 13, 18, 25], "sum_bias_10": [5, 13], "summar": [2, 5, 13], "summari": [2, 18, 19], "support": [2, 3, 4, 5, 10, 12, 23, 28], "supports_bounded_execut": [13, 25], "suppos": [4, 18], "suppress_build_script_link_lin": 15, "sure": 15, "symbol": 2, "sync": 15, "syntax": 4, "system": [1, 5, 15], "t": [4, 13, 24], "t1": 13, "tabl": [0, 1, 2, 5, 6, 8, 10, 17, 19, 21, 23, 29, 30, 32, 33, 35], "table_exist": 1, "table_nam": [6, 8], "table_partition_col": 1, "table_provid": 1, "tableprovid": 36, "tableproviderexport": 1, "tablescan": 3, "tail": 2, "take": [2, 13, 15, 18, 20, 25, 26], "tan": 4, "tangent": 4, "tanh": 4, "target": [1, 5], "target_partit": [1, 5], "task": [21, 30], "technic": 16, "tempor": 22, "temporari": [1, 5], "term": 26, "test": [4, 15], "text": [1, 4], "textual": 21, "th": 4, "than": [2, 3, 4, 5, 18], "thei": [6, 13, 20, 25], "them": [4, 15, 17, 19], "then_expr": 3, "therefor": 2, "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 28, 29, 30, 36], "third": 17, "those": [2, 3, 5, 16, 18, 25, 28], "though": 25, "thread": 16, "three": [25, 26, 28], "through": [16, 22, 28, 29], "thu": 13, "ticket": 15, "tie": 4, "tile": 4, "time": [4, 13, 15, 18, 20, 21, 25, 30], "time64": 4, "timestamp": [4, 21], "tip_amount": [17, 24], "tip_perc": 17, "tips_plus_tol": 24, "tlc": 24, "tmp": [1, 5], "to_arrow_t": 2, "to_hex": 4, "to_panda": [2, 19, 21, 37], "to_polar": 2, "to_proto": [5, 10], "to_pyarrow": [5, 11], "to_pyarrow_dataset": 28, "to_pydict": 2, "to_pylist": 2, "to_str": 36, "to_substrait_plan": 12, "to_timestamp": [4, 21], "to_timestamp_micro": 4, "to_timestamp_milli": 4, "to_timestamp_second": 4, "to_unixtim": 4, "to_val": 4, "to_vari": [3, 5, 10], "togeth": [2, 4, 18], "tolls_amount": 24, "toml": 15, "top": 19, "topic": 28, "total": [2, 4, 16, 17, 20, 21, 28, 29], "total_amount": 17, "total_as_float": 21, "total_as_int": 21, "toward": 4, "tracker": 4, "tradit": 15, "trait": 13, "transact": [1, 5], "transform": [2, 5, 10, 17], "translat": 4, "treat": [3, 26], "treatment": [3, 5], "tree": [5, 10, 20], "trigger": 2, "trim": 4, "trip": [24, 28], "trip_dist": [17, 24], "trivial": [2, 15], "tru": 2, "true": [1, 2, 3, 4, 5, 6, 8, 13, 15, 18, 20, 21, 24, 25, 26, 27], "trunc": 4, "truncat": 4, "try": 1, "trycast": 3, "tupl": [1, 2, 4, 13], "turn": [2, 3, 5], "tutori": 29, "two": [2, 3, 4, 5, 10, 18, 20, 23, 25, 26, 27], "type": [1, 2, 3, 4, 5, 13, 15, 16, 18, 20, 21, 24, 25, 26, 28, 29], "typic": [0, 3, 5, 11, 13, 17], "typing_extens": [5, 11], "u": [18, 20, 28], "udaf": [1, 5, 13, 16, 25], "udaf1": [5, 13], "udaf2": [5, 13], "udaf3": [5, 13], "udf": [1, 5, 14, 16, 25], "udwf": [1, 5, 13], "udwf1": [5, 13], "udwf2": [5, 13], "udwf3": [5, 13], "ultim": 15, "unari": [5, 10], "unbound": [1, 3, 5, 13, 26], "uncompress": 2, "undefin": 15, "underli": [3, 5], "unicod": 4, "union": [2, 3, 4], "union_distinct": 2, "uniqu": [1, 4, 13, 18], "unit": [3, 5, 26], "unixtim": 4, "unknown": 3, "unlik": [3, 5, 26], "unmatch": 23, "unnest": [2, 3], "unnest_column": 2, "unnestexpr": 3, "unoptim": 2, "unspil": [1, 5], "until": 17, "unwrap": 36, "up": [5, 10, 13, 17, 20, 26], "updat": [1, 2, 5, 13, 25], "upon": [4, 25], "upper": 4, "uppercas": 4, "urbango": 23, "url": 1, "us": [1, 2, 3, 4, 5, 6, 7, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 32, 34, 35, 36], "usag": [1, 2, 3, 4, 5, 18, 26], "user": [0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 13, 15, 17, 20, 21, 22, 28, 29], "uses_window_fram": [13, 25], "utc": 4, "utf8": [3, 4, 5], "utf8view": [3, 5], "util": 26, "uuid": 4, "uv": 15, "v": 15, "v4": 4, "valid": [1, 3, 5, 6, 8, 13], "valu": [1, 2, 3, 4, 5, 10, 13, 18, 20, 21, 23, 24, 25, 26], "value1": 4, "value2": 4, "value_i": 4, "value_x": 4, "valueerror": 2, "values_a": 25, "values_b": 25, "var": 4, "var_pop": [4, 18], "var_samp": [4, 18], "var_sampl": 4, "variabl": [1, 4, 5, 15, 20], "varianc": 4, "variant": [3, 5, 10, 23], "variant_nam": [3, 5], "varieti": [20, 25, 28], "variou": 29, "vastli": 15, "vendorid": 24, "venomoth": 26, "venonat": 26, "venu": 21, "venufleur": 21, "venufleurmega": 21, "venusaur": [16, 21, 26, 28, 29], "venusaurmega": [16, 21, 26, 28, 29], "venv": 15, "verbos": 2, "veri": [31, 32], "verifi": 29, "verion": 28, "version": [15, 28], "via": [1, 12, 15, 16, 20, 23, 28, 29, 36, 37], "view": [1, 19, 21, 29], "vink": 26, "virtual": [1, 5, 15], "visual": [5, 10], "volatil": [5, 13, 25], "voltorb": [18, 26], "vulpix": 18, "wa": [1, 5], "wai": [15, 18, 28, 29], "want": [15, 18, 21, 25, 26, 36], "wartortl": [16, 28, 29], "water": [16, 18, 21, 28, 29], "we": [2, 15, 17, 18, 20, 21, 23, 24, 25, 26, 27, 29, 30], "weedl": [16, 26, 28, 29], "weight": [4, 20], "welcom": [15, 29], "well": [1, 5, 15], "were": 20, "what": 17, "whatev": 13, "when": [1, 3, 4, 5, 13, 15, 16, 18, 20, 23, 25, 26, 27, 34], "when_expr": 3, "whenev": 15, "where": [3, 4, 5, 10, 18, 20, 25, 26], "wherea": 20, "whether": [1, 3, 4, 5, 13, 20], "which": [1, 2, 3, 4, 5, 13, 15, 16, 18, 20, 24, 25, 26, 28, 30], "while": [13, 23], "whole": 18, "wide": [20, 27, 28], "window": [1, 3, 4, 5, 13, 18, 22], "window_fram": [3, 4, 5, 26], "windowevalu": [5, 13, 25], "windowexpr": 3, "windowfram": [3, 4, 5, 26], "windowframebound": [3, 5], "windowudf": [1, 5, 13], "with_allow_ddl": [1, 5], "with_allow_dml": [1, 5], "with_allow_stat": [1, 5], "with_batch_s": [1, 5], "with_column": 2, "with_column_renam": 2, "with_create_default_catalog_and_schema": [1, 5, 27], "with_default_catalog_and_schema": [1, 5, 27], "with_disk_manager_dis": [1, 5], "with_disk_manager_o": [1, 5, 27], "with_disk_manager_specifi": [1, 5], "with_fair_spill_pool": [1, 5, 27], "with_greedy_memory_pool": [1, 5], "with_head": 2, "with_information_schema": [1, 5, 27], "with_parquet_prun": [1, 5, 27], "with_repartition_aggreg": [1, 5, 27], "with_repartition_file_min_s": [1, 5], "with_repartition_file_scan": [1, 5], "with_repartition_join": [1, 5, 27], "with_repartition_sort": [1, 5], "with_repartition_window": [1, 5, 27], "with_target_partit": [1, 5, 27], "with_temp_file_path": [1, 5], "with_unbounded_memory_pool": [1, 5], "within": [4, 7, 13, 18, 20], "within_limit": 2, "without": [4, 15, 18, 21, 23, 24, 25], "won": 24, "word": 4, "work": [1, 4, 5, 21, 24, 29], "workflow": 15, "would": [3, 4, 6, 13], "wrap": 2, "wrapper": [5, 11, 15], "write": [2, 6, 12, 25], "write_csv": 2, "write_json": 2, "write_parquet": 2, "written": 16, "x": [4, 13, 16, 21, 26, 28, 29, 37], "xor": 4, "y": [4, 13, 16, 21, 26, 28, 29], "year": 4, "years_in_posit": 20, "yellow": 28, "yellow_tripdata_2021": [17, 24], "yet": 2, "yield": 4, "you": [1, 4, 5, 13, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29, 30, 36], "your": [5, 13, 15, 18, 19, 24, 25, 36], "z": [1, 5, 13], "zero": [2, 4, 16, 30], "zstandard": 2, "zstd": 2, "zubat": 18, "\u03c0": 4}, "titles": ["datafusion.catalog", "datafusion.context", "datafusion.dataframe", "datafusion.expr", "datafusion.functions", "datafusion", "datafusion.input.base", "datafusion.input", "datafusion.input.location", "datafusion.object_store", "datafusion.plan", "datafusion.record_batch", "datafusion.substrait", "datafusion.udf", "API Reference", "Introduction", "DataFusion in Python", "Concepts", "Aggregation", "Basic Operations", "Expressions", "Functions", "Common Operations", "Joins", "Column Selections", "User-Defined Functions", "Window Functions", "Configuration", "Data Sources", "Introduction", "Arrow", "Avro", "CSV", "IO", "JSON", "Parquet", "Custom Table Provider", "SQL"], "titleterms": {"aggreg": [18, 25, 26], "anti": 23, "api": 14, "arrai": 20, "arrow": 30, "attribut": [3, 5, 9, 13], "avail": 26, "avro": 31, "base": 6, "basic": 19, "boolean": 20, "build": 15, "cast": 21, "catalog": 0, "class": [0, 1, 2, 3, 5, 6, 8, 10, 11, 12, 13], "code": 15, "column": [20, 24], "commit": 15, "common": 22, "concept": 17, "condit": 21, "configur": 27, "content": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13], "context": [1, 17], "creat": 28, "csv": 32, "custom": [28, 36], "data": 28, "datafram": [2, 17, 28], "datafus": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 30], "defin": 25, "delta": 28, "depend": 15, "develop": 15, "distinct": 18, "exampl": 16, "export": 30, "expr": 3, "express": [17, 20], "file": 28, "filter": 18, "frame": 26, "from": 30, "full": 23, "function": [4, 5, 18, 20, 21, 25, 26], "guidelin": 15, "hook": 15, "how": 15, "iceberg": 28, "import": 30, "improv": 15, "inner": 23, "input": [6, 7, 8], "instal": [15, 16, 29], "introduct": [15, 29], "io": 33, "join": 23, "json": 34, "lake": 28, "left": 23, "librari": 28, "liter": 20, "local": 28, "locat": 8, "mathemat": 21, "memori": 28, "modul": [0, 1, 2, 3, 4, 6, 8, 9, 10, 11, 12, 13], "null": [18, 26], "object": 28, "object_stor": 9, "oper": [19, 22], "option": 25, "order": [18, 26], "other": [21, 28], "packag": 5, "paramet": [18, 26], "parquet": 35, "partit": 26, "plan": 10, "pre": 15, "provid": [28, 36], "python": [15, 16], "record_batch": 11, "refer": 14, "return": 4, "run": 15, "rust": 15, "scalar": 25, "select": 24, "semi": 23, "separ": 15, "session": 17, "set": [18, 26], "sourc": 28, "speed": 15, "sql": 37, "store": 28, "string": 21, "struct": 20, "submodul": [5, 7], "substrait": 12, "tabl": [28, 36], "tempor": 21, "treatment": [18, 26], "udf": 13, "udwf": 25, "updat": 15, "user": 25, "window": [25, 26]}}) \ No newline at end of file +Search.setIndex({"alltitles": {"API Reference": [[15, null]], "Aggregate Functions": [[20, "aggregate-functions"], [27, "aggregate-functions"], [29, "aggregate-functions"]], "Aggregation": [[20, null]], "Alternative Approach": [[16, "alternative-approach"]], "Arrays": [[22, "arrays"]], "Arrow": [[33, null]], "Attributes": [[3, "attributes"], [5, "attributes"], [10, "attributes"], [14, "attributes"]], "Available Functions": [[29, "available-functions"]], "Avro": [[34, null]], "Basic Operations": [[21, null]], "Boolean": [[22, "boolean"]], "CSV": [[35, null]], "Casting": [[23, "casting"]], "Classes": [[0, "classes"], [1, "classes"], [2, "classes"], [3, "classes"], [5, "classes"], [6, "classes"], [7, "classes"], [8, "classes"], [11, "classes"], [12, "classes"], [13, "classes"], [14, "classes"]], "Column": [[22, "column"]], "Column Selections": [[26, null]], "Common Operations": [[24, null]], "Concepts": [[19, null]], "Conditional": [[23, "conditional"]], "Configuration": [[30, null]], "Create in-memory": [[31, "create-in-memory"]], "Custom Table Provider": [[31, "custom-table-provider"], [39, null]], "Data Sources": [[31, null]], "DataFrame": [[19, "dataframe"]], "DataFusion in Python": [[18, null]], "Delta Lake": [[31, "delta-lake"]], "Distinct": [[20, "distinct"]], "Example": [[18, "example"]], "Exporting from DataFusion": [[33, "exporting-from-datafusion"]], "Expressions": [[19, "expressions"], [22, null]], "Filter": [[20, "filter"]], "Full Join": [[25, "full-join"]], "Functions": [[4, "functions"], [5, "functions"], [9, "functions"], [22, "functions"], [23, null]], "Guidelines for Separating Python and Rust Code": [[17, "guidelines-for-separating-python-and-rust-code"]], "How to develop": [[17, "how-to-develop"]], "IO": [[36, null]], "Iceberg": [[31, "iceberg"]], "Implementation Details": [[16, "implementation-details"]], "Importing to DataFusion": [[33, "importing-to-datafusion"]], "Improving Build Speed": [[17, "improving-build-speed"]], "Inner Join": [[25, "inner-join"]], "Inspiration from Arrow": [[16, "inspiration-from-arrow"]], "Install": [[18, "install"]], "Installation": [[32, "installation"]], "Introduction": [[17, null], [32, null]], "JSON": [[37, null]], "Joins": [[25, null]], "Left Anti Join": [[25, "left-anti-join"]], "Left Join": [[25, "left-join"]], "Left Semi Join": [[25, "left-semi-join"]], "Literal": [[22, "literal"]], "Local file": [[31, "local-file"]], "Mathematical": [[23, "mathematical"]], "Module Contents": [[0, "module-contents"], [1, "module-contents"], [2, "module-contents"], [3, "module-contents"], [4, "module-contents"], [6, "module-contents"], [8, "module-contents"], [9, "module-contents"], [10, "module-contents"], [11, "module-contents"], [12, "module-contents"], [13, "module-contents"], [14, "module-contents"]], "Null Treatment": [[20, "null-treatment"], [29, "null-treatment"]], "Object Store": [[31, "object-store"]], "Ordering": [[20, "ordering"], [29, "ordering"]], "Other": [[23, "other"]], "Other DataFrame Libraries": [[31, "other-dataframe-libraries"]], "Package Contents": [[5, "package-contents"], [7, "package-contents"]], "Parquet": [[38, null]], "Partitions": [[29, "partitions"]], "Python Extensions": [[16, null]], "Registering Views": [[28, null]], "Returns:": [[4, "returns"], [4, "id1"]], "Running & Installing pre-commit hooks": [[17, "running-installing-pre-commit-hooks"]], "SQL": [[40, null]], "Scalar Functions": [[27, "scalar-functions"]], "Session Context": [[19, "session-context"]], "Setting Parameters": [[20, "setting-parameters"], [29, "setting-parameters"]], "Status of Work": [[16, "status-of-work"]], "String": [[23, "string"]], "Structs": [[22, "structs"]], "Submodules": [[5, "submodules"], [7, "submodules"]], "Temporal": [[23, "temporal"]], "The FFI Approach": [[16, "the-ffi-approach"]], "The Primary Issue": [[16, "the-primary-issue"]], "UDWF options": [[27, "udwf-options"]], "Update Dependencies": [[17, "update-dependencies"]], "User-Defined Functions": [[27, null]], "Window Frame": [[29, "window-frame"]], "Window Functions": [[27, "window-functions"], [29, null]], "datafusion": [[5, null]], "datafusion.catalog": [[0, null]], "datafusion.context": [[1, null]], "datafusion.dataframe": [[2, null]], "datafusion.expr": [[3, null]], "datafusion.functions": [[4, null]], "datafusion.input": [[7, null]], "datafusion.input.base": [[6, null]], "datafusion.input.location": [[8, null]], "datafusion.io": [[9, null]], "datafusion.object_store": [[10, null]], "datafusion.plan": [[11, null]], "datafusion.record_batch": [[12, null]], "datafusion.substrait": [[13, null]], "datafusion.udf": [[14, null]]}, "docnames": ["autoapi/datafusion/catalog/index", "autoapi/datafusion/context/index", "autoapi/datafusion/dataframe/index", "autoapi/datafusion/expr/index", "autoapi/datafusion/functions/index", "autoapi/datafusion/index", "autoapi/datafusion/input/base/index", "autoapi/datafusion/input/index", "autoapi/datafusion/input/location/index", "autoapi/datafusion/io/index", "autoapi/datafusion/object_store/index", "autoapi/datafusion/plan/index", "autoapi/datafusion/record_batch/index", "autoapi/datafusion/substrait/index", "autoapi/datafusion/udf/index", "autoapi/index", "contributor-guide/ffi", "contributor-guide/introduction", "index", "user-guide/basics", "user-guide/common-operations/aggregations", "user-guide/common-operations/basic-info", "user-guide/common-operations/expressions", "user-guide/common-operations/functions", "user-guide/common-operations/index", "user-guide/common-operations/joins", "user-guide/common-operations/select-and-filter", "user-guide/common-operations/udf-and-udfa", "user-guide/common-operations/views", "user-guide/common-operations/windows", "user-guide/configuration", "user-guide/data-sources", "user-guide/introduction", "user-guide/io/arrow", "user-guide/io/avro", "user-guide/io/csv", "user-guide/io/index", "user-guide/io/json", "user-guide/io/parquet", "user-guide/io/table_provider", "user-guide/sql"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["autoapi/datafusion/catalog/index.rst", "autoapi/datafusion/context/index.rst", "autoapi/datafusion/dataframe/index.rst", "autoapi/datafusion/expr/index.rst", "autoapi/datafusion/functions/index.rst", "autoapi/datafusion/index.rst", "autoapi/datafusion/input/base/index.rst", "autoapi/datafusion/input/index.rst", "autoapi/datafusion/input/location/index.rst", "autoapi/datafusion/io/index.rst", "autoapi/datafusion/object_store/index.rst", "autoapi/datafusion/plan/index.rst", "autoapi/datafusion/record_batch/index.rst", "autoapi/datafusion/substrait/index.rst", "autoapi/datafusion/udf/index.rst", "autoapi/index.rst", "contributor-guide/ffi.rst", "contributor-guide/introduction.rst", "index.rst", "user-guide/basics.rst", "user-guide/common-operations/aggregations.rst", "user-guide/common-operations/basic-info.rst", "user-guide/common-operations/expressions.rst", "user-guide/common-operations/functions.rst", "user-guide/common-operations/index.rst", "user-guide/common-operations/joins.rst", "user-guide/common-operations/select-and-filter.rst", "user-guide/common-operations/udf-and-udfa.rst", "user-guide/common-operations/views.rst", "user-guide/common-operations/windows.rst", "user-guide/configuration.rst", "user-guide/data-sources.rst", "user-guide/introduction.rst", "user-guide/io/arrow.rst", "user-guide/io/avro.rst", "user-guide/io/csv.rst", "user-guide/io/index.rst", "user-guide/io/json.rst", "user-guide/io/parquet.rst", "user-guide/io/table_provider.rst", "user-guide/sql.rst"], "indexentries": {"__add__() (datafusion.expr method)": [[5, "datafusion.Expr.__add__", false]], "__add__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__add__", false]], "__aiter__() (datafusion.record_batch.recordbatchstream method)": [[12, "datafusion.record_batch.RecordBatchStream.__aiter__", false]], "__aiter__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__aiter__", false]], "__and__() (datafusion.expr method)": [[5, "datafusion.Expr.__and__", false]], "__and__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__and__", false]], "__anext__() (datafusion.record_batch.recordbatchstream method)": [[12, "datafusion.record_batch.RecordBatchStream.__anext__", false]], "__anext__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__anext__", false]], "__arrow_c_array__() (datafusion.context.arrowarrayexportable method)": [[1, "datafusion.context.ArrowArrayExportable.__arrow_c_array__", false]], "__arrow_c_stream__() (datafusion.context.arrowstreamexportable method)": [[1, "datafusion.context.ArrowStreamExportable.__arrow_c_stream__", false]], "__arrow_c_stream__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__arrow_c_stream__", false]], "__call__() (datafusion.aggregateudf method)": [[5, "datafusion.AggregateUDF.__call__", false]], "__call__() (datafusion.scalarudf method)": [[5, "datafusion.ScalarUDF.__call__", false]], "__call__() (datafusion.udf.aggregateudf method)": [[14, "datafusion.udf.AggregateUDF.__call__", false]], "__call__() (datafusion.udf.scalarudf method)": [[14, "datafusion.udf.ScalarUDF.__call__", false]], "__call__() (datafusion.udf.windowudf method)": [[14, "datafusion.udf.WindowUDF.__call__", false]], "__call__() (datafusion.windowudf method)": [[5, "datafusion.WindowUDF.__call__", false]], "__datafusion_table_provider__() (datafusion.context.tableproviderexportable method)": [[1, "datafusion.context.TableProviderExportable.__datafusion_table_provider__", false]], "__eq__() (datafusion.expr method)": [[5, "datafusion.Expr.__eq__", false]], "__eq__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__eq__", false]], "__ge__() (datafusion.expr method)": [[5, "datafusion.Expr.__ge__", false]], "__ge__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__ge__", false]], "__getitem__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__getitem__", false]], "__getitem__() (datafusion.expr method)": [[5, "datafusion.Expr.__getitem__", false]], "__getitem__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__getitem__", false]], "__gt__() (datafusion.expr method)": [[5, "datafusion.Expr.__gt__", false]], "__gt__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__gt__", false]], "__invert__() (datafusion.expr method)": [[5, "datafusion.Expr.__invert__", false]], "__invert__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__invert__", false]], "__iter__() (datafusion.record_batch.recordbatchstream method)": [[12, "datafusion.record_batch.RecordBatchStream.__iter__", false]], "__iter__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__iter__", false]], "__le__() (datafusion.expr method)": [[5, "datafusion.Expr.__le__", false]], "__le__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__le__", false]], "__lt__() (datafusion.expr method)": [[5, "datafusion.Expr.__lt__", false]], "__lt__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__lt__", false]], "__mod__() (datafusion.expr method)": [[5, "datafusion.Expr.__mod__", false]], "__mod__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__mod__", false]], "__mul__() (datafusion.expr method)": [[5, "datafusion.Expr.__mul__", false]], "__mul__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__mul__", false]], "__ne__() (datafusion.expr method)": [[5, "datafusion.Expr.__ne__", false]], "__ne__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__ne__", false]], "__next__() (datafusion.record_batch.recordbatchstream method)": [[12, "datafusion.record_batch.RecordBatchStream.__next__", false]], "__next__() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.__next__", false]], "__or__() (datafusion.expr method)": [[5, "datafusion.Expr.__or__", false]], "__or__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__or__", false]], "__radd__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__radd__", false]], "__radd__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__radd__", false]], "__rand__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rand__", false]], "__rand__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rand__", false]], "__repr__() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.__repr__", false]], "__repr__() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.__repr__", false]], "__repr__() (datafusion.expr method)": [[5, "datafusion.Expr.__repr__", false]], "__repr__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__repr__", false]], "__repr__() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.__repr__", false]], "__repr__() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.__repr__", false]], "__repr__() (datafusion.plan.executionplan method)": [[11, "datafusion.plan.ExecutionPlan.__repr__", false]], "__repr__() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.__repr__", false]], "__richcmp__() (datafusion.expr method)": [[5, "datafusion.Expr.__richcmp__", false]], "__richcmp__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__richcmp__", false]], "__rmod__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rmod__", false]], "__rmod__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rmod__", false]], "__rmul__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rmul__", false]], "__rmul__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rmul__", false]], "__ror__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__ror__", false]], "__ror__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__ror__", false]], "__rsub__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rsub__", false]], "__rsub__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rsub__", false]], "__rtruediv__ (datafusion.expr attribute)": [[5, "datafusion.Expr.__rtruediv__", false]], "__rtruediv__ (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.__rtruediv__", false]], "__str__() (datafusion.udf.volatility method)": [[14, "datafusion.udf.Volatility.__str__", false]], "__sub__() (datafusion.expr method)": [[5, "datafusion.Expr.__sub__", false]], "__sub__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__sub__", false]], "__truediv__() (datafusion.expr method)": [[5, "datafusion.Expr.__truediv__", false]], "__truediv__() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.__truediv__", false]], "_create_window_udf() (datafusion.udf.windowudf static method)": [[14, "datafusion.udf.WindowUDF._create_window_udf", false]], "_create_window_udf() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF._create_window_udf", false]], "_create_window_udf_decorator() (datafusion.udf.windowudf static method)": [[14, "datafusion.udf.WindowUDF._create_window_udf_decorator", false]], "_create_window_udf_decorator() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF._create_window_udf_decorator", false]], "_get_default_name() (datafusion.udf.windowudf static method)": [[14, "datafusion.udf.WindowUDF._get_default_name", false]], "_get_default_name() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF._get_default_name", false]], "_normalize_input_types() (datafusion.udf.windowudf static method)": [[14, "datafusion.udf.WindowUDF._normalize_input_types", false]], "_normalize_input_types() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF._normalize_input_types", false]], "_null_treatment (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._null_treatment", false]], "_order_by (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._order_by", false]], "_partition_by (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._partition_by", false]], "_r (in module datafusion.udf)": [[14, "datafusion.udf._R", false]], "_raw_plan (datafusion.executionplan attribute)": [[5, "datafusion.ExecutionPlan._raw_plan", false]], "_raw_plan (datafusion.logicalplan attribute)": [[5, "datafusion.LogicalPlan._raw_plan", false]], "_raw_plan (datafusion.plan.executionplan attribute)": [[11, "datafusion.plan.ExecutionPlan._raw_plan", false]], "_raw_plan (datafusion.plan.logicalplan attribute)": [[11, "datafusion.plan.LogicalPlan._raw_plan", false]], "_repr_html_() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame._repr_html_", false]], "_to_pyarrow_types (datafusion.expr attribute)": [[5, "datafusion.Expr._to_pyarrow_types", false]], "_to_pyarrow_types (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr._to_pyarrow_types", false]], "_udaf (datafusion.aggregateudf attribute)": [[5, "datafusion.AggregateUDF._udaf", false]], "_udaf (datafusion.udf.aggregateudf attribute)": [[14, "datafusion.udf.AggregateUDF._udaf", false]], "_udf (datafusion.scalarudf attribute)": [[5, "datafusion.ScalarUDF._udf", false]], "_udf (datafusion.udf.scalarudf attribute)": [[14, "datafusion.udf.ScalarUDF._udf", false]], "_udwf (datafusion.udf.windowudf attribute)": [[14, "datafusion.udf.WindowUDF._udwf", false]], "_udwf (datafusion.windowudf attribute)": [[5, "datafusion.WindowUDF._udwf", false]], "_window_frame (datafusion.expr.window attribute)": [[3, "datafusion.expr.Window._window_frame", false]], "abs() (in module datafusion.functions)": [[4, "datafusion.functions.abs", false]], "accumulator (class in datafusion)": [[5, "datafusion.Accumulator", false]], "accumulator (class in datafusion.udf)": [[14, "datafusion.udf.Accumulator", false]], "acos() (in module datafusion.functions)": [[4, "datafusion.functions.acos", false]], "acosh() (in module datafusion.functions)": [[4, "datafusion.functions.acosh", false]], "aggregate (in module datafusion.expr)": [[3, "datafusion.expr.Aggregate", false]], "aggregate() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.aggregate", false]], "aggregatefunction (in module datafusion.expr)": [[3, "datafusion.expr.AggregateFunction", false]], "aggregateudf (class in datafusion)": [[5, "datafusion.AggregateUDF", false]], "aggregateudf (class in datafusion.udf)": [[14, "datafusion.udf.AggregateUDF", false]], "alias (in module datafusion.expr)": [[3, "datafusion.expr.Alias", false]], "alias() (datafusion.expr method)": [[5, "datafusion.Expr.alias", false]], "alias() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.alias", false]], "alias() (in module datafusion.functions)": [[4, "datafusion.functions.alias", false]], "amazons3 (in module datafusion.object_store)": [[10, "datafusion.object_store.AmazonS3", false]], "analyze (in module datafusion.expr)": [[3, "datafusion.expr.Analyze", false]], "approx_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.approx_distinct", false]], "approx_median() (in module datafusion.functions)": [[4, "datafusion.functions.approx_median", false]], "approx_percentile_cont() (in module datafusion.functions)": [[4, "datafusion.functions.approx_percentile_cont", false]], "approx_percentile_cont_with_weight() (in module datafusion.functions)": [[4, "datafusion.functions.approx_percentile_cont_with_weight", false]], "array() (in module datafusion.functions)": [[4, "datafusion.functions.array", false]], "array_agg() (in module datafusion.functions)": [[4, "datafusion.functions.array_agg", false]], "array_append() (in module datafusion.functions)": [[4, "datafusion.functions.array_append", false]], "array_cat() (in module datafusion.functions)": [[4, "datafusion.functions.array_cat", false]], "array_concat() (in module datafusion.functions)": [[4, "datafusion.functions.array_concat", false]], "array_dims() (in module datafusion.functions)": [[4, "datafusion.functions.array_dims", false]], "array_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.array_distinct", false]], "array_element() (in module datafusion.functions)": [[4, "datafusion.functions.array_element", false]], "array_empty() (in module datafusion.functions)": [[4, "datafusion.functions.array_empty", false]], "array_except() (in module datafusion.functions)": [[4, "datafusion.functions.array_except", false]], "array_extract() (in module datafusion.functions)": [[4, "datafusion.functions.array_extract", false]], "array_has() (in module datafusion.functions)": [[4, "datafusion.functions.array_has", false]], "array_has_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_has_all", false]], "array_has_any() (in module datafusion.functions)": [[4, "datafusion.functions.array_has_any", false]], "array_indexof() (in module datafusion.functions)": [[4, "datafusion.functions.array_indexof", false]], "array_intersect() (in module datafusion.functions)": [[4, "datafusion.functions.array_intersect", false]], "array_join() (in module datafusion.functions)": [[4, "datafusion.functions.array_join", false]], "array_length() (in module datafusion.functions)": [[4, "datafusion.functions.array_length", false]], "array_ndims() (in module datafusion.functions)": [[4, "datafusion.functions.array_ndims", false]], "array_pop_back() (in module datafusion.functions)": [[4, "datafusion.functions.array_pop_back", false]], "array_pop_front() (in module datafusion.functions)": [[4, "datafusion.functions.array_pop_front", false]], "array_position() (in module datafusion.functions)": [[4, "datafusion.functions.array_position", false]], "array_positions() (in module datafusion.functions)": [[4, "datafusion.functions.array_positions", false]], "array_prepend() (in module datafusion.functions)": [[4, "datafusion.functions.array_prepend", false]], "array_push_back() (in module datafusion.functions)": [[4, "datafusion.functions.array_push_back", false]], "array_push_front() (in module datafusion.functions)": [[4, "datafusion.functions.array_push_front", false]], "array_remove() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove", false]], "array_remove_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove_all", false]], "array_remove_n() (in module datafusion.functions)": [[4, "datafusion.functions.array_remove_n", false]], "array_repeat() (in module datafusion.functions)": [[4, "datafusion.functions.array_repeat", false]], "array_replace() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace", false]], "array_replace_all() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace_all", false]], "array_replace_n() (in module datafusion.functions)": [[4, "datafusion.functions.array_replace_n", false]], "array_resize() (in module datafusion.functions)": [[4, "datafusion.functions.array_resize", false]], "array_slice() (in module datafusion.functions)": [[4, "datafusion.functions.array_slice", false]], "array_sort() (in module datafusion.functions)": [[4, "datafusion.functions.array_sort", false]], "array_to_string() (in module datafusion.functions)": [[4, "datafusion.functions.array_to_string", false]], "array_union() (in module datafusion.functions)": [[4, "datafusion.functions.array_union", false]], "arrow_cast() (in module datafusion.functions)": [[4, "datafusion.functions.arrow_cast", false]], "arrow_typeof() (in module datafusion.functions)": [[4, "datafusion.functions.arrow_typeof", false]], "arrowarrayexportable (class in datafusion.context)": [[1, "datafusion.context.ArrowArrayExportable", false]], "arrowstreamexportable (class in datafusion.context)": [[1, "datafusion.context.ArrowStreamExportable", false]], "ascending() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.ascending", false]], "ascii() (in module datafusion.functions)": [[4, "datafusion.functions.ascii", false]], "asin() (in module datafusion.functions)": [[4, "datafusion.functions.asin", false]], "asinh() (in module datafusion.functions)": [[4, "datafusion.functions.asinh", false]], "atan() (in module datafusion.functions)": [[4, "datafusion.functions.atan", false]], "atan2() (in module datafusion.functions)": [[4, "datafusion.functions.atan2", false]], "atanh() (in module datafusion.functions)": [[4, "datafusion.functions.atanh", false]], "avg() (in module datafusion.functions)": [[4, "datafusion.functions.avg", false]], "baseinputsource (class in datafusion.input.base)": [[6, "datafusion.input.base.BaseInputSource", false]], "between (in module datafusion.expr)": [[3, "datafusion.expr.Between", false]], "between() (datafusion.expr method)": [[5, "datafusion.Expr.between", false]], "between() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.between", false]], "binaryexpr (in module datafusion.expr)": [[3, "datafusion.expr.BinaryExpr", false]], "bit_and() (in module datafusion.functions)": [[4, "datafusion.functions.bit_and", false]], "bit_length() (in module datafusion.functions)": [[4, "datafusion.functions.bit_length", false]], "bit_or() (in module datafusion.functions)": [[4, "datafusion.functions.bit_or", false]], "bit_xor() (in module datafusion.functions)": [[4, "datafusion.functions.bit_xor", false]], "bool_and() (in module datafusion.functions)": [[4, "datafusion.functions.bool_and", false]], "bool_or() (in module datafusion.functions)": [[4, "datafusion.functions.bool_or", false]], "brotli (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.BROTLI", false]], "btrim() (in module datafusion.functions)": [[4, "datafusion.functions.btrim", false]], "build_table() (datafusion.input.base.baseinputsource method)": [[6, "datafusion.input.base.BaseInputSource.build_table", false]], "build_table() (datafusion.input.location.locationinputplugin method)": [[8, "datafusion.input.location.LocationInputPlugin.build_table", false]], "build_table() (datafusion.input.locationinputplugin method)": [[7, "datafusion.input.LocationInputPlugin.build_table", false]], "cache() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.cache", false]], "canonical_name() (datafusion.expr method)": [[5, "datafusion.Expr.canonical_name", false]], "canonical_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.canonical_name", false]], "cardinality() (in module datafusion.functions)": [[4, "datafusion.functions.cardinality", false]], "case (in module datafusion.expr)": [[3, "datafusion.expr.Case", false]], "case() (in module datafusion.functions)": [[4, "datafusion.functions.case", false]], "case_builder (datafusion.expr.casebuilder attribute)": [[3, "datafusion.expr.CaseBuilder.case_builder", false]], "casebuilder (class in datafusion.expr)": [[3, "datafusion.expr.CaseBuilder", false]], "cast (in module datafusion.expr)": [[3, "datafusion.expr.Cast", false]], "cast() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.cast", false]], "cast() (datafusion.expr method)": [[5, "datafusion.Expr.cast", false]], "cast() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.cast", false]], "catalog (class in datafusion)": [[5, "datafusion.Catalog", false]], "catalog (class in datafusion.catalog)": [[0, "datafusion.catalog.Catalog", false]], "catalog (datafusion.catalog attribute)": [[5, "datafusion.Catalog.catalog", false]], "catalog (datafusion.catalog.catalog attribute)": [[0, "datafusion.catalog.Catalog.catalog", false]], "catalog() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.catalog", false]], "cbrt() (in module datafusion.functions)": [[4, "datafusion.functions.cbrt", false]], "ceil() (in module datafusion.functions)": [[4, "datafusion.functions.ceil", false]], "char_length() (in module datafusion.functions)": [[4, "datafusion.functions.char_length", false]], "character_length() (in module datafusion.functions)": [[4, "datafusion.functions.character_length", false]], "children() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.children", false]], "children() (datafusion.plan.executionplan method)": [[11, "datafusion.plan.ExecutionPlan.children", false]], "chr() (in module datafusion.functions)": [[4, "datafusion.functions.chr", false]], "coalesce() (in module datafusion.functions)": [[4, "datafusion.functions.coalesce", false]], "col() (in module datafusion)": [[5, "datafusion.col", false]], "col() (in module datafusion.functions)": [[4, "datafusion.functions.col", false]], "collect() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.collect", false]], "collect_partitioned() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.collect_partitioned", false]], "column (in module datafusion.expr)": [[3, "datafusion.expr.Column", false]], "column() (datafusion.expr static method)": [[5, "datafusion.Expr.column", false]], "column() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.column", false]], "column() (in module datafusion)": [[5, "datafusion.column", false]], "column_name() (datafusion.expr method)": [[5, "datafusion.Expr.column_name", false]], "column_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.column_name", false]], "compression (class in datafusion.dataframe)": [[2, "datafusion.dataframe.Compression", false]], "concat() (in module datafusion.functions)": [[4, "datafusion.functions.concat", false]], "concat_ws() (in module datafusion.functions)": [[4, "datafusion.functions.concat_ws", false]], "config_internal (datafusion.context.runtimeenvbuilder attribute)": [[1, "datafusion.context.RuntimeEnvBuilder.config_internal", false]], "config_internal (datafusion.context.sessionconfig attribute)": [[1, "datafusion.context.SessionConfig.config_internal", false]], "config_internal (datafusion.runtimeenvbuilder attribute)": [[5, "datafusion.RuntimeEnvBuilder.config_internal", false]], "config_internal (datafusion.sessionconfig attribute)": [[5, "datafusion.SessionConfig.config_internal", false]], "consumer (class in datafusion.substrait)": [[13, "datafusion.substrait.Consumer", false]], "corr() (in module datafusion.functions)": [[4, "datafusion.functions.corr", false]], "cos() (in module datafusion.functions)": [[4, "datafusion.functions.cos", false]], "cosh() (in module datafusion.functions)": [[4, "datafusion.functions.cosh", false]], "cot() (in module datafusion.functions)": [[4, "datafusion.functions.cot", false]], "count() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.count", false]], "count() (in module datafusion.functions)": [[4, "datafusion.functions.count", false]], "count_star() (in module datafusion.functions)": [[4, "datafusion.functions.count_star", false]], "covar() (in module datafusion.functions)": [[4, "datafusion.functions.covar", false]], "covar_pop() (in module datafusion.functions)": [[4, "datafusion.functions.covar_pop", false]], "covar_samp() (in module datafusion.functions)": [[4, "datafusion.functions.covar_samp", false]], "create_dataframe() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.create_dataframe", false]], "create_dataframe_from_logical_plan() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.create_dataframe_from_logical_plan", false]], "creatememorytable (in module datafusion.expr)": [[3, "datafusion.expr.CreateMemoryTable", false]], "createview (in module datafusion.expr)": [[3, "datafusion.expr.CreateView", false]], "ctx (datafusion.context.sessioncontext attribute)": [[1, "datafusion.context.SessionContext.ctx", false]], "cume_dist() (in module datafusion.functions)": [[4, "datafusion.functions.cume_dist", false]], "current_date() (in module datafusion.functions)": [[4, "datafusion.functions.current_date", false]], "current_time() (in module datafusion.functions)": [[4, "datafusion.functions.current_time", false]], "database (class in datafusion)": [[5, "datafusion.Database", false]], "database (class in datafusion.catalog)": [[0, "datafusion.catalog.Database", false]], "database() (datafusion.catalog method)": [[5, "datafusion.Catalog.database", false]], "database() (datafusion.catalog.catalog method)": [[0, "datafusion.catalog.Catalog.database", false]], "dataframe (class in datafusion.dataframe)": [[2, "datafusion.dataframe.DataFrame", false]], "datafusion": [[5, "module-datafusion", false]], "datafusion.catalog": [[0, "module-datafusion.catalog", false]], "datafusion.context": [[1, "module-datafusion.context", false]], "datafusion.dataframe": [[2, "module-datafusion.dataframe", false]], "datafusion.expr": [[3, "module-datafusion.expr", false]], "datafusion.functions": [[4, "module-datafusion.functions", false]], "datafusion.input": [[7, "module-datafusion.input", false]], "datafusion.input.base": [[6, "module-datafusion.input.base", false]], "datafusion.input.location": [[8, "module-datafusion.input.location", false]], "datafusion.io": [[9, "module-datafusion.io", false]], "datafusion.object_store": [[10, "module-datafusion.object_store", false]], "datafusion.plan": [[11, "module-datafusion.plan", false]], "datafusion.record_batch": [[12, "module-datafusion.record_batch", false]], "datafusion.substrait": [[13, "module-datafusion.substrait", false]], "datafusion.udf": [[14, "module-datafusion.udf", false]], "date_bin() (in module datafusion.functions)": [[4, "datafusion.functions.date_bin", false]], "date_part() (in module datafusion.functions)": [[4, "datafusion.functions.date_part", false]], "date_trunc() (in module datafusion.functions)": [[4, "datafusion.functions.date_trunc", false]], "datepart() (in module datafusion.functions)": [[4, "datafusion.functions.datepart", false]], "datetrunc() (in module datafusion.functions)": [[4, "datafusion.functions.datetrunc", false]], "db (datafusion.catalog.database attribute)": [[0, "datafusion.catalog.Database.db", false]], "db (datafusion.database attribute)": [[5, "datafusion.Database.db", false]], "decode() (in module datafusion.functions)": [[4, "datafusion.functions.decode", false]], "degrees() (in module datafusion.functions)": [[4, "datafusion.functions.degrees", false]], "dense_rank() (in module datafusion.functions)": [[4, "datafusion.functions.dense_rank", false]], "deregister_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.deregister_table", false]], "describe() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.describe", false]], "deserialize() (datafusion.substrait.serde static method)": [[13, "datafusion.substrait.Serde.deserialize", false]], "deserialize_bytes() (datafusion.substrait.serde static method)": [[13, "datafusion.substrait.Serde.deserialize_bytes", false]], "df (datafusion.dataframe.dataframe attribute)": [[2, "datafusion.dataframe.DataFrame.df", false]], "dfschema (in module datafusion)": [[5, "datafusion.DFSchema", false]], "digest() (in module datafusion.functions)": [[4, "datafusion.functions.digest", false]], "display() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.display", false]], "display() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display", false]], "display() (datafusion.plan.executionplan method)": [[11, "datafusion.plan.ExecutionPlan.display", false]], "display() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.display", false]], "display_graphviz() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_graphviz", false]], "display_graphviz() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.display_graphviz", false]], "display_indent() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.display_indent", false]], "display_indent() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_indent", false]], "display_indent() (datafusion.plan.executionplan method)": [[11, "datafusion.plan.ExecutionPlan.display_indent", false]], "display_indent() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.display_indent", false]], "display_indent_schema() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.display_indent_schema", false]], "display_indent_schema() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.display_indent_schema", false]], "display_name() (datafusion.expr method)": [[5, "datafusion.Expr.display_name", false]], "display_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.display_name", false]], "distinct (in module datafusion.expr)": [[3, "datafusion.expr.Distinct", false]], "distinct() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.distinct", false]], "distinct() (datafusion.expr method)": [[5, "datafusion.Expr.distinct", false]], "distinct() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.distinct", false]], "drop() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.drop", false]], "droptable (in module datafusion.expr)": [[3, "datafusion.expr.DropTable", false]], "empty() (in module datafusion.functions)": [[4, "datafusion.functions.empty", false]], "empty_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.empty_table", false]], "emptyrelation (in module datafusion.expr)": [[3, "datafusion.expr.EmptyRelation", false]], "enable_url_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.enable_url_table", false]], "encode() (datafusion.substrait.plan method)": [[13, "datafusion.substrait.Plan.encode", false]], "encode() (in module datafusion.functions)": [[4, "datafusion.functions.encode", false]], "end() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.end", false]], "ends_with() (in module datafusion.functions)": [[4, "datafusion.functions.ends_with", false]], "evaluate() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.evaluate", false]], "evaluate() (datafusion.udf.accumulator method)": [[14, "datafusion.udf.Accumulator.evaluate", false]], "evaluate() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.evaluate", false]], "evaluate_all() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.evaluate_all", false]], "evaluate_all_with_rank() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.evaluate_all_with_rank", false]], "except_all() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.except_all", false]], "execute() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.execute", false]], "execute_stream() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execute_stream", false]], "execute_stream_partitioned() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execute_stream_partitioned", false]], "execution_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.execution_plan", false]], "executionplan (class in datafusion)": [[5, "datafusion.ExecutionPlan", false]], "executionplan (class in datafusion.plan)": [[11, "datafusion.plan.ExecutionPlan", false]], "exists (in module datafusion.expr)": [[3, "datafusion.expr.Exists", false]], "exp() (in module datafusion.functions)": [[4, "datafusion.functions.exp", false]], "explain (in module datafusion.expr)": [[3, "datafusion.expr.Explain", false]], "explain() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.explain", false]], "expr (class in datafusion)": [[5, "datafusion.Expr", false]], "expr (class in datafusion.expr)": [[3, "datafusion.expr.Expr", false]], "expr (datafusion.expr attribute)": [[5, "datafusion.Expr.expr", false]], "expr (datafusion.expr.expr attribute)": [[3, "datafusion.expr.Expr.expr", false]], "expr() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.expr", false]], "extension (in module datafusion.expr)": [[3, "datafusion.expr.Extension", false]], "extract() (in module datafusion.functions)": [[4, "datafusion.functions.extract", false]], "factorial() (in module datafusion.functions)": [[4, "datafusion.functions.factorial", false]], "fill_nan() (datafusion.expr method)": [[5, "datafusion.Expr.fill_nan", false]], "fill_nan() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.fill_nan", false]], "fill_null() (datafusion.expr method)": [[5, "datafusion.Expr.fill_null", false]], "fill_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.fill_null", false]], "filter (in module datafusion.expr)": [[3, "datafusion.expr.Filter", false]], "filter() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.filter", false]], "filter() (datafusion.expr method)": [[5, "datafusion.Expr.filter", false]], "filter() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.filter", false]], "find_in_set() (in module datafusion.functions)": [[4, "datafusion.functions.find_in_set", false]], "first_value() (in module datafusion.functions)": [[4, "datafusion.functions.first_value", false]], "flatten() (in module datafusion.functions)": [[4, "datafusion.functions.flatten", false]], "floor() (in module datafusion.functions)": [[4, "datafusion.functions.floor", false]], "frame_bound (datafusion.expr.windowframebound attribute)": [[3, "datafusion.expr.WindowFrameBound.frame_bound", false]], "from_arrow() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_arrow", false]], "from_arrow_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_arrow_table", false]], "from_pandas() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pandas", false]], "from_polars() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_polars", false]], "from_proto() (datafusion.executionplan static method)": [[5, "datafusion.ExecutionPlan.from_proto", false]], "from_proto() (datafusion.logicalplan static method)": [[5, "datafusion.LogicalPlan.from_proto", false]], "from_proto() (datafusion.plan.executionplan static method)": [[11, "datafusion.plan.ExecutionPlan.from_proto", false]], "from_proto() (datafusion.plan.logicalplan static method)": [[11, "datafusion.plan.LogicalPlan.from_proto", false]], "from_pydict() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pydict", false]], "from_pylist() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.from_pylist", false]], "from_str() (datafusion.dataframe.compression class method)": [[2, "datafusion.dataframe.Compression.from_str", false]], "from_substrait_plan() (datafusion.substrait.consumer static method)": [[13, "datafusion.substrait.Consumer.from_substrait_plan", false]], "from_unixtime() (in module datafusion.functions)": [[4, "datafusion.functions.from_unixtime", false]], "gcd() (in module datafusion.functions)": [[4, "datafusion.functions.gcd", false]], "get_default_level() (datafusion.dataframe.compression method)": [[2, "datafusion.dataframe.Compression.get_default_level", false]], "get_frame_units() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_frame_units", false]], "get_frame_units() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_frame_units", false]], "get_lower_bound() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_lower_bound", false]], "get_lower_bound() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_lower_bound", false]], "get_offset() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.get_offset", false]], "get_range() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.get_range", false]], "get_upper_bound() (datafusion.expr.windowframe method)": [[3, "datafusion.expr.WindowFrame.get_upper_bound", false]], "get_upper_bound() (datafusion.windowframe method)": [[5, "datafusion.WindowFrame.get_upper_bound", false]], "global_ctx() (datafusion.context.sessioncontext class method)": [[1, "datafusion.context.SessionContext.global_ctx", false]], "googlecloud (in module datafusion.object_store)": [[10, "datafusion.object_store.GoogleCloud", false]], "groupingset (in module datafusion.expr)": [[3, "datafusion.expr.GroupingSet", false]], "gzip (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.GZIP", false]], "head() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.head", false]], "http (in module datafusion.object_store)": [[10, "datafusion.object_store.Http", false]], "ilike (in module datafusion.expr)": [[3, "datafusion.expr.ILike", false]], "immutable (datafusion.udf.volatility attribute)": [[14, "datafusion.udf.Volatility.Immutable", false]], "in_list() (in module datafusion.functions)": [[4, "datafusion.functions.in_list", false]], "include_rank() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.include_rank", false]], "initcap() (in module datafusion.functions)": [[4, "datafusion.functions.initcap", false]], "inlist (in module datafusion.expr)": [[3, "datafusion.expr.InList", false]], "inputs() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.inputs", false]], "inputs() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.inputs", false]], "insubquery (in module datafusion.expr)": [[3, "datafusion.expr.InSubquery", false]], "intersect() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.intersect", false]], "into_view() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.into_view", false]], "is_causal() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.is_causal", false]], "is_correct_input() (datafusion.input.base.baseinputsource method)": [[6, "datafusion.input.base.BaseInputSource.is_correct_input", false]], "is_correct_input() (datafusion.input.location.locationinputplugin method)": [[8, "datafusion.input.location.LocationInputPlugin.is_correct_input", false]], "is_correct_input() (datafusion.input.locationinputplugin method)": [[7, "datafusion.input.LocationInputPlugin.is_correct_input", false]], "is_current_row() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_current_row", false]], "is_following() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_following", false]], "is_not_null() (datafusion.expr method)": [[5, "datafusion.Expr.is_not_null", false]], "is_not_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.is_not_null", false]], "is_null() (datafusion.expr method)": [[5, "datafusion.Expr.is_null", false]], "is_null() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.is_null", false]], "is_preceding() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_preceding", false]], "is_unbounded() (datafusion.expr.windowframebound method)": [[3, "datafusion.expr.WindowFrameBound.is_unbounded", false]], "isfalse (in module datafusion.expr)": [[3, "datafusion.expr.IsFalse", false]], "isnan() (in module datafusion.functions)": [[4, "datafusion.functions.isnan", false]], "isnotfalse (in module datafusion.expr)": [[3, "datafusion.expr.IsNotFalse", false]], "isnotnull (in module datafusion.expr)": [[3, "datafusion.expr.IsNotNull", false]], "isnottrue (in module datafusion.expr)": [[3, "datafusion.expr.IsNotTrue", false]], "isnotunknown (in module datafusion.expr)": [[3, "datafusion.expr.IsNotUnknown", false]], "isnull (in module datafusion.expr)": [[3, "datafusion.expr.IsNull", false]], "istrue (in module datafusion.expr)": [[3, "datafusion.expr.IsTrue", false]], "isunknown (in module datafusion.expr)": [[3, "datafusion.expr.IsUnknown", false]], "iszero() (in module datafusion.functions)": [[4, "datafusion.functions.iszero", false]], "join (in module datafusion.expr)": [[3, "datafusion.expr.Join", false]], "join() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.join", false]], "join_on() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.join_on", false]], "joinconstraint (in module datafusion.expr)": [[3, "datafusion.expr.JoinConstraint", false]], "jointype (in module datafusion.expr)": [[3, "datafusion.expr.JoinType", false]], "kind (datafusion.catalog.table property)": [[0, "datafusion.catalog.Table.kind", false]], "kind (datafusion.table property)": [[5, "datafusion.Table.kind", false]], "lag() (in module datafusion.functions)": [[4, "datafusion.functions.lag", false]], "last_value() (in module datafusion.functions)": [[4, "datafusion.functions.last_value", false]], "lcm() (in module datafusion.functions)": [[4, "datafusion.functions.lcm", false]], "lead() (in module datafusion.functions)": [[4, "datafusion.functions.lead", false]], "left() (in module datafusion.functions)": [[4, "datafusion.functions.left", false]], "length() (in module datafusion.functions)": [[4, "datafusion.functions.length", false]], "levenshtein() (in module datafusion.functions)": [[4, "datafusion.functions.levenshtein", false]], "like (in module datafusion.expr)": [[3, "datafusion.expr.Like", false]], "limit (in module datafusion.expr)": [[3, "datafusion.expr.Limit", false]], "limit() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.limit", false]], "list_append() (in module datafusion.functions)": [[4, "datafusion.functions.list_append", false]], "list_cat() (in module datafusion.functions)": [[4, "datafusion.functions.list_cat", false]], "list_concat() (in module datafusion.functions)": [[4, "datafusion.functions.list_concat", false]], "list_dims() (in module datafusion.functions)": [[4, "datafusion.functions.list_dims", false]], "list_distinct() (in module datafusion.functions)": [[4, "datafusion.functions.list_distinct", false]], "list_element() (in module datafusion.functions)": [[4, "datafusion.functions.list_element", false]], "list_except() (in module datafusion.functions)": [[4, "datafusion.functions.list_except", false]], "list_extract() (in module datafusion.functions)": [[4, "datafusion.functions.list_extract", false]], "list_indexof() (in module datafusion.functions)": [[4, "datafusion.functions.list_indexof", false]], "list_intersect() (in module datafusion.functions)": [[4, "datafusion.functions.list_intersect", false]], "list_join() (in module datafusion.functions)": [[4, "datafusion.functions.list_join", false]], "list_length() (in module datafusion.functions)": [[4, "datafusion.functions.list_length", false]], "list_ndims() (in module datafusion.functions)": [[4, "datafusion.functions.list_ndims", false]], "list_position() (in module datafusion.functions)": [[4, "datafusion.functions.list_position", false]], "list_positions() (in module datafusion.functions)": [[4, "datafusion.functions.list_positions", false]], "list_prepend() (in module datafusion.functions)": [[4, "datafusion.functions.list_prepend", false]], "list_push_back() (in module datafusion.functions)": [[4, "datafusion.functions.list_push_back", false]], "list_push_front() (in module datafusion.functions)": [[4, "datafusion.functions.list_push_front", false]], "list_remove() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove", false]], "list_remove_all() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove_all", false]], "list_remove_n() (in module datafusion.functions)": [[4, "datafusion.functions.list_remove_n", false]], "list_repeat() (in module datafusion.functions)": [[4, "datafusion.functions.list_repeat", false]], "list_replace() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace", false]], "list_replace_all() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace_all", false]], "list_replace_n() (in module datafusion.functions)": [[4, "datafusion.functions.list_replace_n", false]], "list_resize() (in module datafusion.functions)": [[4, "datafusion.functions.list_resize", false]], "list_slice() (in module datafusion.functions)": [[4, "datafusion.functions.list_slice", false]], "list_sort() (in module datafusion.functions)": [[4, "datafusion.functions.list_sort", false]], "list_to_string() (in module datafusion.functions)": [[4, "datafusion.functions.list_to_string", false]], "list_union() (in module datafusion.functions)": [[4, "datafusion.functions.list_union", false]], "lit() (in module datafusion)": [[5, "datafusion.lit", false]], "literal (in module datafusion.expr)": [[3, "datafusion.expr.Literal", false]], "literal() (datafusion.expr static method)": [[5, "datafusion.Expr.literal", false]], "literal() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.literal", false]], "literal() (in module datafusion)": [[5, "datafusion.literal", false]], "ln() (in module datafusion.functions)": [[4, "datafusion.functions.ln", false]], "localfilesystem (in module datafusion.object_store)": [[10, "datafusion.object_store.LocalFileSystem", false]], "locationinputplugin (class in datafusion.input)": [[7, "datafusion.input.LocationInputPlugin", false]], "locationinputplugin (class in datafusion.input.location)": [[8, "datafusion.input.location.LocationInputPlugin", false]], "log() (in module datafusion.functions)": [[4, "datafusion.functions.log", false]], "log10() (in module datafusion.functions)": [[4, "datafusion.functions.log10", false]], "log2() (in module datafusion.functions)": [[4, "datafusion.functions.log2", false]], "logical_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.logical_plan", false]], "logicalplan (class in datafusion)": [[5, "datafusion.LogicalPlan", false]], "logicalplan (class in datafusion.plan)": [[11, "datafusion.plan.LogicalPlan", false]], "lower() (in module datafusion.functions)": [[4, "datafusion.functions.lower", false]], "lpad() (in module datafusion.functions)": [[4, "datafusion.functions.lpad", false]], "ltrim() (in module datafusion.functions)": [[4, "datafusion.functions.ltrim", false]], "lz4 (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.LZ4", false]], "lz4_raw (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.LZ4_RAW", false]], "make_array() (in module datafusion.functions)": [[4, "datafusion.functions.make_array", false]], "make_date() (in module datafusion.functions)": [[4, "datafusion.functions.make_date", false]], "make_list() (in module datafusion.functions)": [[4, "datafusion.functions.make_list", false]], "max() (in module datafusion.functions)": [[4, "datafusion.functions.max", false]], "md5() (in module datafusion.functions)": [[4, "datafusion.functions.md5", false]], "mean() (in module datafusion.functions)": [[4, "datafusion.functions.mean", false]], "median() (in module datafusion.functions)": [[4, "datafusion.functions.median", false]], "memoize() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.memoize", false]], "merge() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.merge", false]], "merge() (datafusion.udf.accumulator method)": [[14, "datafusion.udf.Accumulator.merge", false]], "microsoftazure (in module datafusion.object_store)": [[10, "datafusion.object_store.MicrosoftAzure", false]], "min() (in module datafusion.functions)": [[4, "datafusion.functions.min", false]], "module": [[0, "module-datafusion.catalog", false], [1, "module-datafusion.context", false], [2, "module-datafusion.dataframe", false], [3, "module-datafusion.expr", false], [4, "module-datafusion.functions", false], [5, "module-datafusion", false], [6, "module-datafusion.input.base", false], [7, "module-datafusion.input", false], [8, "module-datafusion.input.location", false], [9, "module-datafusion.io", false], [10, "module-datafusion.object_store", false], [11, "module-datafusion.plan", false], [12, "module-datafusion.record_batch", false], [13, "module-datafusion.substrait", false], [14, "module-datafusion.udf", false]], "named_struct() (in module datafusion.functions)": [[4, "datafusion.functions.named_struct", false]], "names() (datafusion.catalog method)": [[5, "datafusion.Catalog.names", false]], "names() (datafusion.catalog.catalog method)": [[0, "datafusion.catalog.Catalog.names", false]], "names() (datafusion.catalog.database method)": [[0, "datafusion.catalog.Database.names", false]], "names() (datafusion.database method)": [[5, "datafusion.Database.names", false]], "nanvl() (in module datafusion.functions)": [[4, "datafusion.functions.nanvl", false]], "negative (in module datafusion.expr)": [[3, "datafusion.expr.Negative", false]], "next() (datafusion.record_batch.recordbatchstream method)": [[12, "datafusion.record_batch.RecordBatchStream.next", false]], "next() (datafusion.recordbatchstream method)": [[5, "datafusion.RecordBatchStream.next", false]], "not (in module datafusion.expr)": [[3, "datafusion.expr.Not", false]], "now() (in module datafusion.functions)": [[4, "datafusion.functions.now", false]], "nth_value() (in module datafusion.functions)": [[4, "datafusion.functions.nth_value", false]], "ntile() (in module datafusion.functions)": [[4, "datafusion.functions.ntile", false]], "null_treatment() (datafusion.expr method)": [[5, "datafusion.Expr.null_treatment", false]], "null_treatment() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.null_treatment", false]], "nullif() (in module datafusion.functions)": [[4, "datafusion.functions.nullif", false]], "nulls_first() (datafusion.expr.sortexpr method)": [[3, "datafusion.expr.SortExpr.nulls_first", false]], "nvl() (in module datafusion.functions)": [[4, "datafusion.functions.nvl", false]], "octet_length() (in module datafusion.functions)": [[4, "datafusion.functions.octet_length", false]], "optimized_logical_plan() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.optimized_logical_plan", false]], "options_internal (datafusion.context.sqloptions attribute)": [[1, "datafusion.context.SQLOptions.options_internal", false]], "options_internal (datafusion.sqloptions attribute)": [[5, "datafusion.SQLOptions.options_internal", false]], "order_by() (datafusion.expr method)": [[5, "datafusion.Expr.order_by", false]], "order_by() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.order_by", false]], "order_by() (in module datafusion.functions)": [[4, "datafusion.functions.order_by", false]], "otherwise() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.otherwise", false]], "over() (datafusion.expr method)": [[5, "datafusion.Expr.over", false]], "over() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.over", false]], "overlay() (in module datafusion.functions)": [[4, "datafusion.functions.overlay", false]], "partition_by() (datafusion.expr method)": [[5, "datafusion.Expr.partition_by", false]], "partition_by() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.partition_by", false]], "partition_count (datafusion.executionplan property)": [[5, "datafusion.ExecutionPlan.partition_count", false]], "partition_count (datafusion.plan.executionplan property)": [[11, "datafusion.plan.ExecutionPlan.partition_count", false]], "partitioning (in module datafusion.expr)": [[3, "datafusion.expr.Partitioning", false]], "percent_rank() (in module datafusion.functions)": [[4, "datafusion.functions.percent_rank", false]], "pi() (in module datafusion.functions)": [[4, "datafusion.functions.pi", false]], "placeholder (in module datafusion.expr)": [[3, "datafusion.expr.Placeholder", false]], "plan (class in datafusion.substrait)": [[13, "datafusion.substrait.Plan", false]], "plan_internal (datafusion.substrait.plan attribute)": [[13, "datafusion.substrait.Plan.plan_internal", false]], "pow() (in module datafusion.functions)": [[4, "datafusion.functions.pow", false]], "power() (in module datafusion.functions)": [[4, "datafusion.functions.power", false]], "producer (class in datafusion.substrait)": [[13, "datafusion.substrait.Producer", false]], "projection (in module datafusion.expr)": [[3, "datafusion.expr.Projection", false]], "python_value() (datafusion.expr method)": [[5, "datafusion.Expr.python_value", false]], "python_value() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.python_value", false]], "radians() (in module datafusion.functions)": [[4, "datafusion.functions.radians", false]], "random() (in module datafusion.functions)": [[4, "datafusion.functions.random", false]], "range() (in module datafusion.functions)": [[4, "datafusion.functions.range", false]], "rank() (in module datafusion.functions)": [[4, "datafusion.functions.rank", false]], "raw_sort (datafusion.expr.sortexpr attribute)": [[3, "datafusion.expr.SortExpr.raw_sort", false]], "rbs (datafusion.record_batch.recordbatchstream attribute)": [[12, "datafusion.record_batch.RecordBatchStream.rbs", false]], "rbs (datafusion.recordbatchstream attribute)": [[5, "datafusion.RecordBatchStream.rbs", false]], "read_avro() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_avro", false]], "read_avro() (in module datafusion)": [[5, "datafusion.read_avro", false]], "read_avro() (in module datafusion.io)": [[9, "datafusion.io.read_avro", false]], "read_csv() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_csv", false]], "read_csv() (in module datafusion)": [[5, "datafusion.read_csv", false]], "read_csv() (in module datafusion.io)": [[9, "datafusion.io.read_csv", false]], "read_json() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_json", false]], "read_json() (in module datafusion)": [[5, "datafusion.read_json", false]], "read_json() (in module datafusion.io)": [[9, "datafusion.io.read_json", false]], "read_parquet() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_parquet", false]], "read_parquet() (in module datafusion)": [[5, "datafusion.read_parquet", false]], "read_parquet() (in module datafusion.io)": [[9, "datafusion.io.read_parquet", false]], "read_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.read_table", false]], "record_batch (datafusion.record_batch.recordbatch attribute)": [[12, "datafusion.record_batch.RecordBatch.record_batch", false]], "record_batch (datafusion.recordbatch attribute)": [[5, "datafusion.RecordBatch.record_batch", false]], "recordbatch (class in datafusion)": [[5, "datafusion.RecordBatch", false]], "recordbatch (class in datafusion.record_batch)": [[12, "datafusion.record_batch.RecordBatch", false]], "recordbatchstream (class in datafusion)": [[5, "datafusion.RecordBatchStream", false]], "recordbatchstream (class in datafusion.record_batch)": [[12, "datafusion.record_batch.RecordBatchStream", false]], "regexp_count() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_count", false]], "regexp_like() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_like", false]], "regexp_match() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_match", false]], "regexp_replace() (in module datafusion.functions)": [[4, "datafusion.functions.regexp_replace", false]], "register_avro() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_avro", false]], "register_csv() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_csv", false]], "register_dataset() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_dataset", false]], "register_json() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_json", false]], "register_listing_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_listing_table", false]], "register_object_store() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_object_store", false]], "register_parquet() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_parquet", false]], "register_record_batches() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_record_batches", false]], "register_table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_table", false]], "register_table_provider() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_table_provider", false]], "register_udaf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udaf", false]], "register_udf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udf", false]], "register_udwf() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_udwf", false]], "register_view() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.register_view", false]], "regr_avgx() (in module datafusion.functions)": [[4, "datafusion.functions.regr_avgx", false]], "regr_avgy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_avgy", false]], "regr_count() (in module datafusion.functions)": [[4, "datafusion.functions.regr_count", false]], "regr_intercept() (in module datafusion.functions)": [[4, "datafusion.functions.regr_intercept", false]], "regr_r2() (in module datafusion.functions)": [[4, "datafusion.functions.regr_r2", false]], "regr_slope() (in module datafusion.functions)": [[4, "datafusion.functions.regr_slope", false]], "regr_sxx() (in module datafusion.functions)": [[4, "datafusion.functions.regr_sxx", false]], "regr_sxy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_sxy", false]], "regr_syy() (in module datafusion.functions)": [[4, "datafusion.functions.regr_syy", false]], "repartition (in module datafusion.expr)": [[3, "datafusion.expr.Repartition", false]], "repartition() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.repartition", false]], "repartition_by_hash() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.repartition_by_hash", false]], "repeat() (in module datafusion.functions)": [[4, "datafusion.functions.repeat", false]], "replace() (in module datafusion.functions)": [[4, "datafusion.functions.replace", false]], "reverse() (in module datafusion.functions)": [[4, "datafusion.functions.reverse", false]], "rex_call_operands() (datafusion.expr method)": [[5, "datafusion.Expr.rex_call_operands", false]], "rex_call_operands() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_call_operands", false]], "rex_call_operator() (datafusion.expr method)": [[5, "datafusion.Expr.rex_call_operator", false]], "rex_call_operator() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_call_operator", false]], "rex_type() (datafusion.expr method)": [[5, "datafusion.Expr.rex_type", false]], "rex_type() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.rex_type", false]], "right() (in module datafusion.functions)": [[4, "datafusion.functions.right", false]], "round() (in module datafusion.functions)": [[4, "datafusion.functions.round", false]], "row_number() (in module datafusion.functions)": [[4, "datafusion.functions.row_number", false]], "rpad() (in module datafusion.functions)": [[4, "datafusion.functions.rpad", false]], "rtrim() (in module datafusion.functions)": [[4, "datafusion.functions.rtrim", false]], "runtimeconfig (class in datafusion.context)": [[1, "datafusion.context.RuntimeConfig", false]], "runtimeenvbuilder (class in datafusion)": [[5, "datafusion.RuntimeEnvBuilder", false]], "runtimeenvbuilder (class in datafusion.context)": [[1, "datafusion.context.RuntimeEnvBuilder", false]], "scalarsubquery (in module datafusion.expr)": [[3, "datafusion.expr.ScalarSubquery", false]], "scalarudf (class in datafusion)": [[5, "datafusion.ScalarUDF", false]], "scalarudf (class in datafusion.udf)": [[14, "datafusion.udf.ScalarUDF", false]], "scalarvariable (in module datafusion.expr)": [[3, "datafusion.expr.ScalarVariable", false]], "schema (datafusion.catalog.table property)": [[0, "datafusion.catalog.Table.schema", false]], "schema (datafusion.table property)": [[5, "datafusion.Table.schema", false]], "schema() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.schema", false]], "schema_name() (datafusion.expr method)": [[5, "datafusion.Expr.schema_name", false]], "schema_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.schema_name", false]], "select() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.select", false]], "select_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.select_columns", false]], "serde (class in datafusion.substrait)": [[13, "datafusion.substrait.Serde", false]], "serialize() (datafusion.substrait.serde static method)": [[13, "datafusion.substrait.Serde.serialize", false]], "serialize_bytes() (datafusion.substrait.serde static method)": [[13, "datafusion.substrait.Serde.serialize_bytes", false]], "serialize_to_plan() (datafusion.substrait.serde static method)": [[13, "datafusion.substrait.Serde.serialize_to_plan", false]], "session_id() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.session_id", false]], "sessionconfig (class in datafusion)": [[5, "datafusion.SessionConfig", false]], "sessionconfig (class in datafusion.context)": [[1, "datafusion.context.SessionConfig", false]], "sessioncontext (class in datafusion.context)": [[1, "datafusion.context.SessionContext", false]], "set() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.set", false]], "set() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.set", false]], "sha224() (in module datafusion.functions)": [[4, "datafusion.functions.sha224", false]], "sha256() (in module datafusion.functions)": [[4, "datafusion.functions.sha256", false]], "sha384() (in module datafusion.functions)": [[4, "datafusion.functions.sha384", false]], "sha512() (in module datafusion.functions)": [[4, "datafusion.functions.sha512", false]], "show() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.show", false]], "signum() (in module datafusion.functions)": [[4, "datafusion.functions.signum", false]], "similarto (in module datafusion.expr)": [[3, "datafusion.expr.SimilarTo", false]], "sin() (in module datafusion.functions)": [[4, "datafusion.functions.sin", false]], "sinh() (in module datafusion.functions)": [[4, "datafusion.functions.sinh", false]], "snappy (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.SNAPPY", false]], "sort (in module datafusion.expr)": [[3, "datafusion.expr.Sort", false]], "sort() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.sort", false]], "sort() (datafusion.expr method)": [[5, "datafusion.Expr.sort", false]], "sort() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.sort", false]], "sortexpr (class in datafusion.expr)": [[3, "datafusion.expr.SortExpr", false]], "split_part() (in module datafusion.functions)": [[4, "datafusion.functions.split_part", false]], "sql() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.sql", false]], "sql_with_options() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.sql_with_options", false]], "sqloptions (class in datafusion)": [[5, "datafusion.SQLOptions", false]], "sqloptions (class in datafusion.context)": [[1, "datafusion.context.SQLOptions", false]], "sqrt() (in module datafusion.functions)": [[4, "datafusion.functions.sqrt", false]], "stable (datafusion.udf.volatility attribute)": [[14, "datafusion.udf.Volatility.Stable", false]], "starts_with() (in module datafusion.functions)": [[4, "datafusion.functions.starts_with", false]], "state() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.state", false]], "state() (datafusion.udf.accumulator method)": [[14, "datafusion.udf.Accumulator.state", false]], "stddev() (in module datafusion.functions)": [[4, "datafusion.functions.stddev", false]], "stddev_pop() (in module datafusion.functions)": [[4, "datafusion.functions.stddev_pop", false]], "stddev_samp() (in module datafusion.functions)": [[4, "datafusion.functions.stddev_samp", false]], "string_agg() (in module datafusion.functions)": [[4, "datafusion.functions.string_agg", false]], "string_literal() (datafusion.expr static method)": [[5, "datafusion.Expr.string_literal", false]], "string_literal() (datafusion.expr.expr static method)": [[3, "datafusion.expr.Expr.string_literal", false]], "strpos() (in module datafusion.functions)": [[4, "datafusion.functions.strpos", false]], "struct() (in module datafusion.functions)": [[4, "datafusion.functions.struct", false]], "subquery (in module datafusion.expr)": [[3, "datafusion.expr.Subquery", false]], "subqueryalias (in module datafusion.expr)": [[3, "datafusion.expr.SubqueryAlias", false]], "substr() (in module datafusion.functions)": [[4, "datafusion.functions.substr", false]], "substr_index() (in module datafusion.functions)": [[4, "datafusion.functions.substr_index", false]], "substring() (in module datafusion.functions)": [[4, "datafusion.functions.substring", false]], "sum() (in module datafusion.functions)": [[4, "datafusion.functions.sum", false]], "supports_bounded_execution() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.supports_bounded_execution", false]], "table (class in datafusion)": [[5, "datafusion.Table", false]], "table (class in datafusion.catalog)": [[0, "datafusion.catalog.Table", false]], "table (datafusion.catalog.table attribute)": [[0, "datafusion.catalog.Table.table", false]], "table (datafusion.table attribute)": [[5, "datafusion.Table.table", false]], "table() (datafusion.catalog.database method)": [[0, "datafusion.catalog.Database.table", false]], "table() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.table", false]], "table() (datafusion.database method)": [[5, "datafusion.Database.table", false]], "table_exist() (datafusion.context.sessioncontext method)": [[1, "datafusion.context.SessionContext.table_exist", false]], "tableproviderexportable (class in datafusion.context)": [[1, "datafusion.context.TableProviderExportable", false]], "tablescan (in module datafusion.expr)": [[3, "datafusion.expr.TableScan", false]], "tail() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.tail", false]], "tan() (in module datafusion.functions)": [[4, "datafusion.functions.tan", false]], "tanh() (in module datafusion.functions)": [[4, "datafusion.functions.tanh", false]], "to_arrow_table() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_arrow_table", false]], "to_hex() (in module datafusion.functions)": [[4, "datafusion.functions.to_hex", false]], "to_pandas() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pandas", false]], "to_polars() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_polars", false]], "to_proto() (datafusion.executionplan method)": [[5, "datafusion.ExecutionPlan.to_proto", false]], "to_proto() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.to_proto", false]], "to_proto() (datafusion.plan.executionplan method)": [[11, "datafusion.plan.ExecutionPlan.to_proto", false]], "to_proto() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.to_proto", false]], "to_pyarrow() (datafusion.record_batch.recordbatch method)": [[12, "datafusion.record_batch.RecordBatch.to_pyarrow", false]], "to_pyarrow() (datafusion.recordbatch method)": [[5, "datafusion.RecordBatch.to_pyarrow", false]], "to_pydict() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pydict", false]], "to_pylist() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.to_pylist", false]], "to_substrait_plan() (datafusion.substrait.producer static method)": [[13, "datafusion.substrait.Producer.to_substrait_plan", false]], "to_timestamp() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp", false]], "to_timestamp_micros() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_micros", false]], "to_timestamp_millis() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_millis", false]], "to_timestamp_nanos() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_nanos", false]], "to_timestamp_seconds() (in module datafusion.functions)": [[4, "datafusion.functions.to_timestamp_seconds", false]], "to_unixtime() (in module datafusion.functions)": [[4, "datafusion.functions.to_unixtime", false]], "to_variant() (datafusion.expr method)": [[5, "datafusion.Expr.to_variant", false]], "to_variant() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.to_variant", false]], "to_variant() (datafusion.logicalplan method)": [[5, "datafusion.LogicalPlan.to_variant", false]], "to_variant() (datafusion.plan.logicalplan method)": [[11, "datafusion.plan.LogicalPlan.to_variant", false]], "transform() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.transform", false]], "translate() (in module datafusion.functions)": [[4, "datafusion.functions.translate", false]], "trim() (in module datafusion.functions)": [[4, "datafusion.functions.trim", false]], "trunc() (in module datafusion.functions)": [[4, "datafusion.functions.trunc", false]], "trycast (in module datafusion.expr)": [[3, "datafusion.expr.TryCast", false]], "types() (datafusion.expr method)": [[5, "datafusion.Expr.types", false]], "types() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.types", false]], "udaf (in module datafusion)": [[5, "datafusion.udaf", false]], "udaf (in module datafusion.udf)": [[14, "datafusion.udf.udaf", false]], "udaf() (datafusion.aggregateudf static method)": [[5, "datafusion.AggregateUDF.udaf", false]], "udaf() (datafusion.udf.aggregateudf static method)": [[14, "datafusion.udf.AggregateUDF.udaf", false]], "udf (in module datafusion)": [[5, "datafusion.udf", false]], "udf (in module datafusion.udf)": [[14, "datafusion.udf.udf", false]], "udf() (datafusion.scalarudf static method)": [[5, "datafusion.ScalarUDF.udf", false]], "udf() (datafusion.udf.scalarudf static method)": [[14, "datafusion.udf.ScalarUDF.udf", false]], "udwf (in module datafusion)": [[5, "datafusion.udwf", false]], "udwf (in module datafusion.udf)": [[14, "datafusion.udf.udwf", false]], "udwf() (datafusion.udf.windowudf static method)": [[14, "datafusion.udf.WindowUDF.udwf", false]], "udwf() (datafusion.windowudf static method)": [[5, "datafusion.WindowUDF.udwf", false]], "uncompressed (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.UNCOMPRESSED", false]], "union (in module datafusion.expr)": [[3, "datafusion.expr.Union", false]], "union() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.union", false]], "union_distinct() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.union_distinct", false]], "unnest (in module datafusion.expr)": [[3, "datafusion.expr.Unnest", false]], "unnest_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.unnest_columns", false]], "unnestexpr (in module datafusion.expr)": [[3, "datafusion.expr.UnnestExpr", false]], "update() (datafusion.accumulator method)": [[5, "datafusion.Accumulator.update", false]], "update() (datafusion.udf.accumulator method)": [[14, "datafusion.udf.Accumulator.update", false]], "upper() (in module datafusion.functions)": [[4, "datafusion.functions.upper", false]], "uses_window_frame() (datafusion.udf.windowevaluator method)": [[14, "datafusion.udf.WindowEvaluator.uses_window_frame", false]], "uuid() (in module datafusion.functions)": [[4, "datafusion.functions.uuid", false]], "var() (in module datafusion.functions)": [[4, "datafusion.functions.var", false]], "var_pop() (in module datafusion.functions)": [[4, "datafusion.functions.var_pop", false]], "var_samp() (in module datafusion.functions)": [[4, "datafusion.functions.var_samp", false]], "var_sample() (in module datafusion.functions)": [[4, "datafusion.functions.var_sample", false]], "variant_name() (datafusion.expr method)": [[5, "datafusion.Expr.variant_name", false]], "variant_name() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.variant_name", false]], "volatile (datafusion.udf.volatility attribute)": [[14, "datafusion.udf.Volatility.Volatile", false]], "volatility (class in datafusion.udf)": [[14, "datafusion.udf.Volatility", false]], "when() (datafusion.expr.casebuilder method)": [[3, "datafusion.expr.CaseBuilder.when", false]], "when() (in module datafusion.functions)": [[4, "datafusion.functions.when", false]], "window (class in datafusion.expr)": [[3, "datafusion.expr.Window", false]], "window() (in module datafusion.functions)": [[4, "datafusion.functions.window", false]], "window_frame (datafusion.expr.windowframe attribute)": [[3, "datafusion.expr.WindowFrame.window_frame", false]], "window_frame (datafusion.windowframe attribute)": [[5, "datafusion.WindowFrame.window_frame", false]], "window_frame() (datafusion.expr method)": [[5, "datafusion.Expr.window_frame", false]], "window_frame() (datafusion.expr.expr method)": [[3, "datafusion.expr.Expr.window_frame", false]], "windowevaluator (class in datafusion.udf)": [[14, "datafusion.udf.WindowEvaluator", false]], "windowexpr (in module datafusion.expr)": [[3, "datafusion.expr.WindowExpr", false]], "windowframe (class in datafusion)": [[5, "datafusion.WindowFrame", false]], "windowframe (class in datafusion.expr)": [[3, "datafusion.expr.WindowFrame", false]], "windowframebound (class in datafusion.expr)": [[3, "datafusion.expr.WindowFrameBound", false]], "windowudf (class in datafusion)": [[5, "datafusion.WindowUDF", false]], "windowudf (class in datafusion.udf)": [[14, "datafusion.udf.WindowUDF", false]], "with_allow_ddl() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_ddl", false]], "with_allow_ddl() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_ddl", false]], "with_allow_dml() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_dml", false]], "with_allow_dml() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_dml", false]], "with_allow_statements() (datafusion.context.sqloptions method)": [[1, "datafusion.context.SQLOptions.with_allow_statements", false]], "with_allow_statements() (datafusion.sqloptions method)": [[5, "datafusion.SQLOptions.with_allow_statements", false]], "with_batch_size() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_batch_size", false]], "with_batch_size() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_batch_size", false]], "with_column() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_column", false]], "with_column_renamed() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_column_renamed", false]], "with_columns() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.with_columns", false]], "with_create_default_catalog_and_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_create_default_catalog_and_schema", false]], "with_create_default_catalog_and_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_create_default_catalog_and_schema", false]], "with_default_catalog_and_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_default_catalog_and_schema", false]], "with_default_catalog_and_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_default_catalog_and_schema", false]], "with_disk_manager_disabled() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_disabled", false]], "with_disk_manager_disabled() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_disabled", false]], "with_disk_manager_os() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_os", false]], "with_disk_manager_os() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_os", false]], "with_disk_manager_specified() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_disk_manager_specified", false]], "with_disk_manager_specified() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_disk_manager_specified", false]], "with_fair_spill_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_fair_spill_pool", false]], "with_fair_spill_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_fair_spill_pool", false]], "with_greedy_memory_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_greedy_memory_pool", false]], "with_greedy_memory_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_greedy_memory_pool", false]], "with_information_schema() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_information_schema", false]], "with_information_schema() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_information_schema", false]], "with_parquet_pruning() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_parquet_pruning", false]], "with_parquet_pruning() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_parquet_pruning", false]], "with_repartition_aggregations() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_aggregations", false]], "with_repartition_aggregations() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_aggregations", false]], "with_repartition_file_min_size() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_file_min_size", false]], "with_repartition_file_min_size() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_file_min_size", false]], "with_repartition_file_scans() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_file_scans", false]], "with_repartition_file_scans() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_file_scans", false]], "with_repartition_joins() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_joins", false]], "with_repartition_joins() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_joins", false]], "with_repartition_sorts() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_sorts", false]], "with_repartition_sorts() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_sorts", false]], "with_repartition_windows() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_repartition_windows", false]], "with_repartition_windows() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_repartition_windows", false]], "with_target_partitions() (datafusion.context.sessionconfig method)": [[1, "datafusion.context.SessionConfig.with_target_partitions", false]], "with_target_partitions() (datafusion.sessionconfig method)": [[5, "datafusion.SessionConfig.with_target_partitions", false]], "with_temp_file_path() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_temp_file_path", false]], "with_temp_file_path() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_temp_file_path", false]], "with_unbounded_memory_pool() (datafusion.context.runtimeenvbuilder method)": [[1, "datafusion.context.RuntimeEnvBuilder.with_unbounded_memory_pool", false]], "with_unbounded_memory_pool() (datafusion.runtimeenvbuilder method)": [[5, "datafusion.RuntimeEnvBuilder.with_unbounded_memory_pool", false]], "write_csv() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_csv", false]], "write_json() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_json", false]], "write_parquet() (datafusion.dataframe.dataframe method)": [[2, "datafusion.dataframe.DataFrame.write_parquet", false]], "zstd (datafusion.dataframe.compression attribute)": [[2, "datafusion.dataframe.Compression.ZSTD", false]]}, "objects": {"": [[5, 0, 0, "-", "datafusion"]], "datafusion": [[5, 1, 1, "", "Accumulator"], [5, 1, 1, "", "AggregateUDF"], [5, 1, 1, "", "Catalog"], [5, 4, 1, "", "DFSchema"], [5, 1, 1, "", "Database"], [5, 1, 1, "", "ExecutionPlan"], [5, 1, 1, "", "Expr"], [5, 1, 1, "", "LogicalPlan"], [5, 1, 1, "", "RecordBatch"], [5, 1, 1, "", "RecordBatchStream"], [5, 1, 1, "", "RuntimeEnvBuilder"], [5, 1, 1, "", "SQLOptions"], [5, 1, 1, "", "ScalarUDF"], [5, 1, 1, "", "SessionConfig"], [5, 1, 1, "", "Table"], [5, 1, 1, "", "WindowFrame"], [5, 1, 1, "", "WindowUDF"], [0, 0, 0, "-", "catalog"], [5, 6, 1, "", "col"], [5, 6, 1, "", "column"], [1, 0, 0, "-", "context"], [2, 0, 0, "-", "dataframe"], [3, 0, 0, "-", "expr"], [4, 0, 0, "-", "functions"], [7, 0, 0, "-", "input"], [9, 0, 0, "-", "io"], [5, 6, 1, "", "lit"], [5, 6, 1, "", "literal"], [10, 0, 0, "-", "object_store"], [11, 0, 0, "-", "plan"], [5, 6, 1, "", "read_avro"], [5, 6, 1, "", "read_csv"], [5, 6, 1, "", "read_json"], [5, 6, 1, "", "read_parquet"], [12, 0, 0, "-", "record_batch"], [13, 0, 0, "-", "substrait"], [5, 4, 1, "", "udaf"], [14, 0, 0, "-", "udf"], [5, 4, 1, "", "udwf"]], "datafusion.Accumulator": [[5, 2, 1, "", "evaluate"], [5, 2, 1, "", "merge"], [5, 2, 1, "", "state"], [5, 2, 1, "", "update"]], "datafusion.AggregateUDF": [[5, 2, 1, "", "__call__"], [5, 3, 1, "", "_udaf"], [5, 2, 1, "", "udaf"]], "datafusion.Catalog": [[5, 3, 1, "", "catalog"], [5, 2, 1, "", "database"], [5, 2, 1, "", "names"]], "datafusion.Database": [[5, 3, 1, "", "db"], [5, 2, 1, "", "names"], [5, 2, 1, "", "table"]], "datafusion.ExecutionPlan": [[5, 2, 1, "", "__repr__"], [5, 3, 1, "", "_raw_plan"], [5, 2, 1, "", "children"], [5, 2, 1, "", "display"], [5, 2, 1, "", "display_indent"], [5, 2, 1, "", "from_proto"], [5, 5, 1, "", "partition_count"], [5, 2, 1, "", "to_proto"]], "datafusion.Expr": [[5, 2, 1, "", "__add__"], [5, 2, 1, "", "__and__"], [5, 2, 1, "", "__eq__"], [5, 2, 1, "", "__ge__"], [5, 2, 1, "", "__getitem__"], [5, 2, 1, "", "__gt__"], [5, 2, 1, "", "__invert__"], [5, 2, 1, "", "__le__"], [5, 2, 1, "", "__lt__"], [5, 2, 1, "", "__mod__"], [5, 2, 1, "", "__mul__"], [5, 2, 1, "", "__ne__"], [5, 2, 1, "", "__or__"], [5, 3, 1, "", "__radd__"], [5, 3, 1, "", "__rand__"], [5, 2, 1, "", "__repr__"], [5, 2, 1, "", "__richcmp__"], [5, 3, 1, "", "__rmod__"], [5, 3, 1, "", "__rmul__"], [5, 3, 1, "", "__ror__"], [5, 3, 1, "", "__rsub__"], [5, 3, 1, "", "__rtruediv__"], [5, 2, 1, "", "__sub__"], [5, 2, 1, "", "__truediv__"], [5, 3, 1, "", "_to_pyarrow_types"], [5, 2, 1, "", "alias"], [5, 2, 1, "", "between"], [5, 2, 1, "", "canonical_name"], [5, 2, 1, "", "cast"], [5, 2, 1, "", "column"], [5, 2, 1, "", "column_name"], [5, 2, 1, "", "display_name"], [5, 2, 1, "", "distinct"], [5, 3, 1, "", "expr"], [5, 2, 1, "", "fill_nan"], [5, 2, 1, "", "fill_null"], [5, 2, 1, "", "filter"], [5, 2, 1, "", "is_not_null"], [5, 2, 1, "", "is_null"], [5, 2, 1, "", "literal"], [5, 2, 1, "", "null_treatment"], [5, 2, 1, "", "order_by"], [5, 2, 1, "", "over"], [5, 2, 1, "", "partition_by"], [5, 2, 1, "", "python_value"], [5, 2, 1, "", "rex_call_operands"], [5, 2, 1, "", "rex_call_operator"], [5, 2, 1, "", "rex_type"], [5, 2, 1, "", "schema_name"], [5, 2, 1, "", "sort"], [5, 2, 1, "", "string_literal"], [5, 2, 1, "", "to_variant"], [5, 2, 1, "", "types"], [5, 2, 1, "", "variant_name"], [5, 2, 1, "", "window_frame"]], "datafusion.LogicalPlan": [[5, 2, 1, "", "__repr__"], [5, 3, 1, "", "_raw_plan"], [5, 2, 1, "", "display"], [5, 2, 1, "", "display_graphviz"], [5, 2, 1, "", "display_indent"], [5, 2, 1, "", "display_indent_schema"], [5, 2, 1, "", "from_proto"], [5, 2, 1, "", "inputs"], [5, 2, 1, "", "to_proto"], [5, 2, 1, "", "to_variant"]], "datafusion.RecordBatch": [[5, 3, 1, "", "record_batch"], [5, 2, 1, "", "to_pyarrow"]], "datafusion.RecordBatchStream": [[5, 2, 1, "", "__aiter__"], [5, 2, 1, "", "__anext__"], [5, 2, 1, "", "__iter__"], [5, 2, 1, "", "__next__"], [5, 2, 1, "", "next"], [5, 3, 1, "", "rbs"]], "datafusion.RuntimeEnvBuilder": [[5, 3, 1, "", "config_internal"], [5, 2, 1, "", "with_disk_manager_disabled"], [5, 2, 1, "", "with_disk_manager_os"], [5, 2, 1, "", "with_disk_manager_specified"], [5, 2, 1, "", "with_fair_spill_pool"], [5, 2, 1, "", "with_greedy_memory_pool"], [5, 2, 1, "", "with_temp_file_path"], [5, 2, 1, "", "with_unbounded_memory_pool"]], "datafusion.SQLOptions": [[5, 3, 1, "", "options_internal"], [5, 2, 1, "", "with_allow_ddl"], [5, 2, 1, "", "with_allow_dml"], [5, 2, 1, "", "with_allow_statements"]], "datafusion.ScalarUDF": [[5, 2, 1, "", "__call__"], [5, 3, 1, "", "_udf"], [5, 2, 1, "", "udf"]], "datafusion.SessionConfig": [[5, 3, 1, "", "config_internal"], [5, 2, 1, "", "set"], [5, 2, 1, "", "with_batch_size"], [5, 2, 1, "", "with_create_default_catalog_and_schema"], [5, 2, 1, "", "with_default_catalog_and_schema"], [5, 2, 1, "", "with_information_schema"], [5, 2, 1, "", "with_parquet_pruning"], [5, 2, 1, "", "with_repartition_aggregations"], [5, 2, 1, "", "with_repartition_file_min_size"], [5, 2, 1, "", "with_repartition_file_scans"], [5, 2, 1, "", "with_repartition_joins"], [5, 2, 1, "", "with_repartition_sorts"], [5, 2, 1, "", "with_repartition_windows"], [5, 2, 1, "", "with_target_partitions"]], "datafusion.Table": [[5, 5, 1, "", "kind"], [5, 5, 1, "", "schema"], [5, 3, 1, "", "table"]], "datafusion.WindowFrame": [[5, 2, 1, "", "get_frame_units"], [5, 2, 1, "", "get_lower_bound"], [5, 2, 1, "", "get_upper_bound"], [5, 3, 1, "", "window_frame"]], "datafusion.WindowUDF": [[5, 2, 1, "", "__call__"], [5, 2, 1, "", "_create_window_udf"], [5, 2, 1, "", "_create_window_udf_decorator"], [5, 2, 1, "", "_get_default_name"], [5, 2, 1, "", "_normalize_input_types"], [5, 3, 1, "", "_udwf"], [5, 2, 1, "", "udwf"]], "datafusion.catalog": [[0, 1, 1, "", "Catalog"], [0, 1, 1, "", "Database"], [0, 1, 1, "", "Table"]], "datafusion.catalog.Catalog": [[0, 3, 1, "", "catalog"], [0, 2, 1, "", "database"], [0, 2, 1, "", "names"]], "datafusion.catalog.Database": [[0, 3, 1, "", "db"], [0, 2, 1, "", "names"], [0, 2, 1, "", "table"]], "datafusion.catalog.Table": [[0, 5, 1, "", "kind"], [0, 5, 1, "", "schema"], [0, 3, 1, "", "table"]], "datafusion.context": [[1, 1, 1, "", "ArrowArrayExportable"], [1, 1, 1, "", "ArrowStreamExportable"], [1, 1, 1, "", "RuntimeConfig"], [1, 1, 1, "", "RuntimeEnvBuilder"], [1, 1, 1, "", "SQLOptions"], [1, 1, 1, "", "SessionConfig"], [1, 1, 1, "", "SessionContext"], [1, 1, 1, "", "TableProviderExportable"]], "datafusion.context.ArrowArrayExportable": [[1, 2, 1, "", "__arrow_c_array__"]], "datafusion.context.ArrowStreamExportable": [[1, 2, 1, "", "__arrow_c_stream__"]], "datafusion.context.RuntimeEnvBuilder": [[1, 3, 1, "", "config_internal"], [1, 2, 1, "", "with_disk_manager_disabled"], [1, 2, 1, "", "with_disk_manager_os"], [1, 2, 1, "", "with_disk_manager_specified"], [1, 2, 1, "", "with_fair_spill_pool"], [1, 2, 1, "", "with_greedy_memory_pool"], [1, 2, 1, "", "with_temp_file_path"], [1, 2, 1, "", "with_unbounded_memory_pool"]], "datafusion.context.SQLOptions": [[1, 3, 1, "", "options_internal"], [1, 2, 1, "", "with_allow_ddl"], [1, 2, 1, "", "with_allow_dml"], [1, 2, 1, "", "with_allow_statements"]], "datafusion.context.SessionConfig": [[1, 3, 1, "", "config_internal"], [1, 2, 1, "", "set"], [1, 2, 1, "", "with_batch_size"], [1, 2, 1, "", "with_create_default_catalog_and_schema"], [1, 2, 1, "", "with_default_catalog_and_schema"], [1, 2, 1, "", "with_information_schema"], [1, 2, 1, "", "with_parquet_pruning"], [1, 2, 1, "", "with_repartition_aggregations"], [1, 2, 1, "", "with_repartition_file_min_size"], [1, 2, 1, "", "with_repartition_file_scans"], [1, 2, 1, "", "with_repartition_joins"], [1, 2, 1, "", "with_repartition_sorts"], [1, 2, 1, "", "with_repartition_windows"], [1, 2, 1, "", "with_target_partitions"]], "datafusion.context.SessionContext": [[1, 2, 1, "", "catalog"], [1, 2, 1, "", "create_dataframe"], [1, 2, 1, "", "create_dataframe_from_logical_plan"], [1, 3, 1, "", "ctx"], [1, 2, 1, "", "deregister_table"], [1, 2, 1, "", "empty_table"], [1, 2, 1, "", "enable_url_table"], [1, 2, 1, "", "execute"], [1, 2, 1, "", "from_arrow"], [1, 2, 1, "", "from_arrow_table"], [1, 2, 1, "", "from_pandas"], [1, 2, 1, "", "from_polars"], [1, 2, 1, "", "from_pydict"], [1, 2, 1, "", "from_pylist"], [1, 2, 1, "", "global_ctx"], [1, 2, 1, "", "read_avro"], [1, 2, 1, "", "read_csv"], [1, 2, 1, "", "read_json"], [1, 2, 1, "", "read_parquet"], [1, 2, 1, "", "read_table"], [1, 2, 1, "", "register_avro"], [1, 2, 1, "", "register_csv"], [1, 2, 1, "", "register_dataset"], [1, 2, 1, "", "register_json"], [1, 2, 1, "", "register_listing_table"], [1, 2, 1, "", "register_object_store"], [1, 2, 1, "", "register_parquet"], [1, 2, 1, "", "register_record_batches"], [1, 2, 1, "", "register_table"], [1, 2, 1, "", "register_table_provider"], [1, 2, 1, "", "register_udaf"], [1, 2, 1, "", "register_udf"], [1, 2, 1, "", "register_udwf"], [1, 2, 1, "", "register_view"], [1, 2, 1, "", "session_id"], [1, 2, 1, "", "sql"], [1, 2, 1, "", "sql_with_options"], [1, 2, 1, "", "table"], [1, 2, 1, "", "table_exist"]], "datafusion.context.TableProviderExportable": [[1, 2, 1, "", "__datafusion_table_provider__"]], "datafusion.dataframe": [[2, 1, 1, "", "Compression"], [2, 1, 1, "", "DataFrame"]], "datafusion.dataframe.Compression": [[2, 3, 1, "", "BROTLI"], [2, 3, 1, "", "GZIP"], [2, 3, 1, "", "LZ4"], [2, 3, 1, "", "LZ4_RAW"], [2, 3, 1, "", "SNAPPY"], [2, 3, 1, "", "UNCOMPRESSED"], [2, 3, 1, "", "ZSTD"], [2, 2, 1, "", "from_str"], [2, 2, 1, "", "get_default_level"]], "datafusion.dataframe.DataFrame": [[2, 2, 1, "", "__arrow_c_stream__"], [2, 2, 1, "", "__getitem__"], [2, 2, 1, "", "__repr__"], [2, 2, 1, "", "_repr_html_"], [2, 2, 1, "", "aggregate"], [2, 2, 1, "", "cache"], [2, 2, 1, "", "cast"], [2, 2, 1, "", "collect"], [2, 2, 1, "", "collect_partitioned"], [2, 2, 1, "", "count"], [2, 2, 1, "", "describe"], [2, 3, 1, "", "df"], [2, 2, 1, "", "distinct"], [2, 2, 1, "", "drop"], [2, 2, 1, "", "except_all"], [2, 2, 1, "", "execute_stream"], [2, 2, 1, "", "execute_stream_partitioned"], [2, 2, 1, "", "execution_plan"], [2, 2, 1, "", "explain"], [2, 2, 1, "", "filter"], [2, 2, 1, "", "head"], [2, 2, 1, "", "intersect"], [2, 2, 1, "", "into_view"], [2, 2, 1, "", "join"], [2, 2, 1, "", "join_on"], [2, 2, 1, "", "limit"], [2, 2, 1, "", "logical_plan"], [2, 2, 1, "", "optimized_logical_plan"], [2, 2, 1, "", "repartition"], [2, 2, 1, "", "repartition_by_hash"], [2, 2, 1, "", "schema"], [2, 2, 1, "", "select"], [2, 2, 1, "", "select_columns"], [2, 2, 1, "", "show"], [2, 2, 1, "", "sort"], [2, 2, 1, "", "tail"], [2, 2, 1, "", "to_arrow_table"], [2, 2, 1, "", "to_pandas"], [2, 2, 1, "", "to_polars"], [2, 2, 1, "", "to_pydict"], [2, 2, 1, "", "to_pylist"], [2, 2, 1, "", "transform"], [2, 2, 1, "", "union"], [2, 2, 1, "", "union_distinct"], [2, 2, 1, "", "unnest_columns"], [2, 2, 1, "", "with_column"], [2, 2, 1, "", "with_column_renamed"], [2, 2, 1, "", "with_columns"], [2, 2, 1, "", "write_csv"], [2, 2, 1, "", "write_json"], [2, 2, 1, "", "write_parquet"]], "datafusion.expr": [[3, 4, 1, "", "Aggregate"], [3, 4, 1, "", "AggregateFunction"], [3, 4, 1, "", "Alias"], [3, 4, 1, "", "Analyze"], [3, 4, 1, "", "Between"], [3, 4, 1, "", "BinaryExpr"], [3, 4, 1, "", "Case"], [3, 1, 1, "", "CaseBuilder"], [3, 4, 1, "", "Cast"], [3, 4, 1, "", "Column"], [3, 4, 1, "", "CreateMemoryTable"], [3, 4, 1, "", "CreateView"], [3, 4, 1, "", "Distinct"], [3, 4, 1, "", "DropTable"], [3, 4, 1, "", "EmptyRelation"], [3, 4, 1, "", "Exists"], [3, 4, 1, "", "Explain"], [3, 1, 1, "", "Expr"], [3, 4, 1, "", "Extension"], [3, 4, 1, "", "Filter"], [3, 4, 1, "", "GroupingSet"], [3, 4, 1, "", "ILike"], [3, 4, 1, "", "InList"], [3, 4, 1, "", "InSubquery"], [3, 4, 1, "", "IsFalse"], [3, 4, 1, "", "IsNotFalse"], [3, 4, 1, "", "IsNotNull"], [3, 4, 1, "", "IsNotTrue"], [3, 4, 1, "", "IsNotUnknown"], [3, 4, 1, "", "IsNull"], [3, 4, 1, "", "IsTrue"], [3, 4, 1, "", "IsUnknown"], [3, 4, 1, "", "Join"], [3, 4, 1, "", "JoinConstraint"], [3, 4, 1, "", "JoinType"], [3, 4, 1, "", "Like"], [3, 4, 1, "", "Limit"], [3, 4, 1, "", "Literal"], [3, 4, 1, "", "Negative"], [3, 4, 1, "", "Not"], [3, 4, 1, "", "Partitioning"], [3, 4, 1, "", "Placeholder"], [3, 4, 1, "", "Projection"], [3, 4, 1, "", "Repartition"], [3, 4, 1, "", "ScalarSubquery"], [3, 4, 1, "", "ScalarVariable"], [3, 4, 1, "", "SimilarTo"], [3, 4, 1, "", "Sort"], [3, 1, 1, "", "SortExpr"], [3, 4, 1, "", "Subquery"], [3, 4, 1, "", "SubqueryAlias"], [3, 4, 1, "", "TableScan"], [3, 4, 1, "", "TryCast"], [3, 4, 1, "", "Union"], [3, 4, 1, "", "Unnest"], [3, 4, 1, "", "UnnestExpr"], [3, 1, 1, "", "Window"], [3, 4, 1, "", "WindowExpr"], [3, 1, 1, "", "WindowFrame"], [3, 1, 1, "", "WindowFrameBound"]], "datafusion.expr.CaseBuilder": [[3, 3, 1, "", "case_builder"], [3, 2, 1, "", "end"], [3, 2, 1, "", "otherwise"], [3, 2, 1, "", "when"]], "datafusion.expr.Expr": [[3, 2, 1, "", "__add__"], [3, 2, 1, "", "__and__"], [3, 2, 1, "", "__eq__"], [3, 2, 1, "", "__ge__"], [3, 2, 1, "", "__getitem__"], [3, 2, 1, "", "__gt__"], [3, 2, 1, "", "__invert__"], [3, 2, 1, "", "__le__"], [3, 2, 1, "", "__lt__"], [3, 2, 1, "", "__mod__"], [3, 2, 1, "", "__mul__"], [3, 2, 1, "", "__ne__"], [3, 2, 1, "", "__or__"], [3, 3, 1, "", "__radd__"], [3, 3, 1, "", "__rand__"], [3, 2, 1, "", "__repr__"], [3, 2, 1, "", "__richcmp__"], [3, 3, 1, "", "__rmod__"], [3, 3, 1, "", "__rmul__"], [3, 3, 1, "", "__ror__"], [3, 3, 1, "", "__rsub__"], [3, 3, 1, "", "__rtruediv__"], [3, 2, 1, "", "__sub__"], [3, 2, 1, "", "__truediv__"], [3, 3, 1, "", "_to_pyarrow_types"], [3, 2, 1, "", "alias"], [3, 2, 1, "", "between"], [3, 2, 1, "", "canonical_name"], [3, 2, 1, "", "cast"], [3, 2, 1, "", "column"], [3, 2, 1, "", "column_name"], [3, 2, 1, "", "display_name"], [3, 2, 1, "", "distinct"], [3, 3, 1, "", "expr"], [3, 2, 1, "", "fill_nan"], [3, 2, 1, "", "fill_null"], [3, 2, 1, "", "filter"], [3, 2, 1, "", "is_not_null"], [3, 2, 1, "", "is_null"], [3, 2, 1, "", "literal"], [3, 2, 1, "", "null_treatment"], [3, 2, 1, "", "order_by"], [3, 2, 1, "", "over"], [3, 2, 1, "", "partition_by"], [3, 2, 1, "", "python_value"], [3, 2, 1, "", "rex_call_operands"], [3, 2, 1, "", "rex_call_operator"], [3, 2, 1, "", "rex_type"], [3, 2, 1, "", "schema_name"], [3, 2, 1, "", "sort"], [3, 2, 1, "", "string_literal"], [3, 2, 1, "", "to_variant"], [3, 2, 1, "", "types"], [3, 2, 1, "", "variant_name"], [3, 2, 1, "", "window_frame"]], "datafusion.expr.SortExpr": [[3, 2, 1, "", "__repr__"], [3, 2, 1, "", "ascending"], [3, 2, 1, "", "expr"], [3, 2, 1, "", "nulls_first"], [3, 3, 1, "", "raw_sort"]], "datafusion.expr.Window": [[3, 3, 1, "", "_null_treatment"], [3, 3, 1, "", "_order_by"], [3, 3, 1, "", "_partition_by"], [3, 3, 1, "", "_window_frame"]], "datafusion.expr.WindowFrame": [[3, 2, 1, "", "get_frame_units"], [3, 2, 1, "", "get_lower_bound"], [3, 2, 1, "", "get_upper_bound"], [3, 3, 1, "", "window_frame"]], "datafusion.expr.WindowFrameBound": [[3, 3, 1, "", "frame_bound"], [3, 2, 1, "", "get_offset"], [3, 2, 1, "", "is_current_row"], [3, 2, 1, "", "is_following"], [3, 2, 1, "", "is_preceding"], [3, 2, 1, "", "is_unbounded"]], "datafusion.functions": [[4, 6, 1, "", "abs"], [4, 6, 1, "", "acos"], [4, 6, 1, "", "acosh"], [4, 6, 1, "", "alias"], [4, 6, 1, "", "approx_distinct"], [4, 6, 1, "", "approx_median"], [4, 6, 1, "", "approx_percentile_cont"], [4, 6, 1, "", "approx_percentile_cont_with_weight"], [4, 6, 1, "", "array"], [4, 6, 1, "", "array_agg"], [4, 6, 1, "", "array_append"], [4, 6, 1, "", "array_cat"], [4, 6, 1, "", "array_concat"], [4, 6, 1, "", "array_dims"], [4, 6, 1, "", "array_distinct"], [4, 6, 1, "", "array_element"], [4, 6, 1, "", "array_empty"], [4, 6, 1, "", "array_except"], [4, 6, 1, "", "array_extract"], [4, 6, 1, "", "array_has"], [4, 6, 1, "", "array_has_all"], [4, 6, 1, "", "array_has_any"], [4, 6, 1, "", "array_indexof"], [4, 6, 1, "", "array_intersect"], [4, 6, 1, "", "array_join"], [4, 6, 1, "", "array_length"], [4, 6, 1, "", "array_ndims"], [4, 6, 1, "", "array_pop_back"], [4, 6, 1, "", "array_pop_front"], [4, 6, 1, "", "array_position"], [4, 6, 1, "", "array_positions"], [4, 6, 1, "", "array_prepend"], [4, 6, 1, "", "array_push_back"], [4, 6, 1, "", "array_push_front"], [4, 6, 1, "", "array_remove"], [4, 6, 1, "", "array_remove_all"], [4, 6, 1, "", "array_remove_n"], [4, 6, 1, "", "array_repeat"], [4, 6, 1, "", "array_replace"], [4, 6, 1, "", "array_replace_all"], [4, 6, 1, "", "array_replace_n"], [4, 6, 1, "", "array_resize"], [4, 6, 1, "", "array_slice"], [4, 6, 1, "", "array_sort"], [4, 6, 1, "", "array_to_string"], [4, 6, 1, "", "array_union"], [4, 6, 1, "", "arrow_cast"], [4, 6, 1, "", "arrow_typeof"], [4, 6, 1, "", "ascii"], [4, 6, 1, "", "asin"], [4, 6, 1, "", "asinh"], [4, 6, 1, "", "atan"], [4, 6, 1, "", "atan2"], [4, 6, 1, "", "atanh"], [4, 6, 1, "", "avg"], [4, 6, 1, "", "bit_and"], [4, 6, 1, "", "bit_length"], [4, 6, 1, "", "bit_or"], [4, 6, 1, "", "bit_xor"], [4, 6, 1, "", "bool_and"], [4, 6, 1, "", "bool_or"], [4, 6, 1, "", "btrim"], [4, 6, 1, "", "cardinality"], [4, 6, 1, "", "case"], [4, 6, 1, "", "cbrt"], [4, 6, 1, "", "ceil"], [4, 6, 1, "", "char_length"], [4, 6, 1, "", "character_length"], [4, 6, 1, "", "chr"], [4, 6, 1, "", "coalesce"], [4, 6, 1, "", "col"], [4, 6, 1, "", "concat"], [4, 6, 1, "", "concat_ws"], [4, 6, 1, "", "corr"], [4, 6, 1, "", "cos"], [4, 6, 1, "", "cosh"], [4, 6, 1, "", "cot"], [4, 6, 1, "", "count"], [4, 6, 1, "", "count_star"], [4, 6, 1, "", "covar"], [4, 6, 1, "", "covar_pop"], [4, 6, 1, "", "covar_samp"], [4, 6, 1, "", "cume_dist"], [4, 6, 1, "", "current_date"], [4, 6, 1, "", "current_time"], [4, 6, 1, "", "date_bin"], [4, 6, 1, "", "date_part"], [4, 6, 1, "", "date_trunc"], [4, 6, 1, "", "datepart"], [4, 6, 1, "", "datetrunc"], [4, 6, 1, "", "decode"], [4, 6, 1, "", "degrees"], [4, 6, 1, "", "dense_rank"], [4, 6, 1, "", "digest"], [4, 6, 1, "", "empty"], [4, 6, 1, "", "encode"], [4, 6, 1, "", "ends_with"], [4, 6, 1, "", "exp"], [4, 6, 1, "", "extract"], [4, 6, 1, "", "factorial"], [4, 6, 1, "", "find_in_set"], [4, 6, 1, "", "first_value"], [4, 6, 1, "", "flatten"], [4, 6, 1, "", "floor"], [4, 6, 1, "", "from_unixtime"], [4, 6, 1, "", "gcd"], [4, 6, 1, "", "in_list"], [4, 6, 1, "", "initcap"], [4, 6, 1, "", "isnan"], [4, 6, 1, "", "iszero"], [4, 6, 1, "", "lag"], [4, 6, 1, "", "last_value"], [4, 6, 1, "", "lcm"], [4, 6, 1, "", "lead"], [4, 6, 1, "", "left"], [4, 6, 1, "", "length"], [4, 6, 1, "", "levenshtein"], [4, 6, 1, "", "list_append"], [4, 6, 1, "", "list_cat"], [4, 6, 1, "", "list_concat"], [4, 6, 1, "", "list_dims"], [4, 6, 1, "", "list_distinct"], [4, 6, 1, "", "list_element"], [4, 6, 1, "", "list_except"], [4, 6, 1, "", "list_extract"], [4, 6, 1, "", "list_indexof"], [4, 6, 1, "", "list_intersect"], [4, 6, 1, "", "list_join"], [4, 6, 1, "", "list_length"], [4, 6, 1, "", "list_ndims"], [4, 6, 1, "", "list_position"], [4, 6, 1, "", "list_positions"], [4, 6, 1, "", "list_prepend"], [4, 6, 1, "", "list_push_back"], [4, 6, 1, "", "list_push_front"], [4, 6, 1, "", "list_remove"], [4, 6, 1, "", "list_remove_all"], [4, 6, 1, "", "list_remove_n"], [4, 6, 1, "", "list_repeat"], [4, 6, 1, "", "list_replace"], [4, 6, 1, "", "list_replace_all"], [4, 6, 1, "", "list_replace_n"], [4, 6, 1, "", "list_resize"], [4, 6, 1, "", "list_slice"], [4, 6, 1, "", "list_sort"], [4, 6, 1, "", "list_to_string"], [4, 6, 1, "", "list_union"], [4, 6, 1, "", "ln"], [4, 6, 1, "", "log"], [4, 6, 1, "", "log10"], [4, 6, 1, "", "log2"], [4, 6, 1, "", "lower"], [4, 6, 1, "", "lpad"], [4, 6, 1, "", "ltrim"], [4, 6, 1, "", "make_array"], [4, 6, 1, "", "make_date"], [4, 6, 1, "", "make_list"], [4, 6, 1, "", "max"], [4, 6, 1, "", "md5"], [4, 6, 1, "", "mean"], [4, 6, 1, "", "median"], [4, 6, 1, "", "min"], [4, 6, 1, "", "named_struct"], [4, 6, 1, "", "nanvl"], [4, 6, 1, "", "now"], [4, 6, 1, "", "nth_value"], [4, 6, 1, "", "ntile"], [4, 6, 1, "", "nullif"], [4, 6, 1, "", "nvl"], [4, 6, 1, "", "octet_length"], [4, 6, 1, "", "order_by"], [4, 6, 1, "", "overlay"], [4, 6, 1, "", "percent_rank"], [4, 6, 1, "", "pi"], [4, 6, 1, "", "pow"], [4, 6, 1, "", "power"], [4, 6, 1, "", "radians"], [4, 6, 1, "", "random"], [4, 6, 1, "", "range"], [4, 6, 1, "", "rank"], [4, 6, 1, "", "regexp_count"], [4, 6, 1, "", "regexp_like"], [4, 6, 1, "", "regexp_match"], [4, 6, 1, "", "regexp_replace"], [4, 6, 1, "", "regr_avgx"], [4, 6, 1, "", "regr_avgy"], [4, 6, 1, "", "regr_count"], [4, 6, 1, "", "regr_intercept"], [4, 6, 1, "", "regr_r2"], [4, 6, 1, "", "regr_slope"], [4, 6, 1, "", "regr_sxx"], [4, 6, 1, "", "regr_sxy"], [4, 6, 1, "", "regr_syy"], [4, 6, 1, "", "repeat"], [4, 6, 1, "", "replace"], [4, 6, 1, "", "reverse"], [4, 6, 1, "", "right"], [4, 6, 1, "", "round"], [4, 6, 1, "", "row_number"], [4, 6, 1, "", "rpad"], [4, 6, 1, "", "rtrim"], [4, 6, 1, "", "sha224"], [4, 6, 1, "", "sha256"], [4, 6, 1, "", "sha384"], [4, 6, 1, "", "sha512"], [4, 6, 1, "", "signum"], [4, 6, 1, "", "sin"], [4, 6, 1, "", "sinh"], [4, 6, 1, "", "split_part"], [4, 6, 1, "", "sqrt"], [4, 6, 1, "", "starts_with"], [4, 6, 1, "", "stddev"], [4, 6, 1, "", "stddev_pop"], [4, 6, 1, "", "stddev_samp"], [4, 6, 1, "", "string_agg"], [4, 6, 1, "", "strpos"], [4, 6, 1, "", "struct"], [4, 6, 1, "", "substr"], [4, 6, 1, "", "substr_index"], [4, 6, 1, "", "substring"], [4, 6, 1, "", "sum"], [4, 6, 1, "", "tan"], [4, 6, 1, "", "tanh"], [4, 6, 1, "", "to_hex"], [4, 6, 1, "", "to_timestamp"], [4, 6, 1, "", "to_timestamp_micros"], [4, 6, 1, "", "to_timestamp_millis"], [4, 6, 1, "", "to_timestamp_nanos"], [4, 6, 1, "", "to_timestamp_seconds"], [4, 6, 1, "", "to_unixtime"], [4, 6, 1, "", "translate"], [4, 6, 1, "", "trim"], [4, 6, 1, "", "trunc"], [4, 6, 1, "", "upper"], [4, 6, 1, "", "uuid"], [4, 6, 1, "", "var"], [4, 6, 1, "", "var_pop"], [4, 6, 1, "", "var_samp"], [4, 6, 1, "", "var_sample"], [4, 6, 1, "", "when"], [4, 6, 1, "", "window"]], "datafusion.input": [[7, 1, 1, "", "LocationInputPlugin"], [6, 0, 0, "-", "base"], [8, 0, 0, "-", "location"]], "datafusion.input.LocationInputPlugin": [[7, 2, 1, "", "build_table"], [7, 2, 1, "", "is_correct_input"]], "datafusion.input.base": [[6, 1, 1, "", "BaseInputSource"]], "datafusion.input.base.BaseInputSource": [[6, 2, 1, "", "build_table"], [6, 2, 1, "", "is_correct_input"]], "datafusion.input.location": [[8, 1, 1, "", "LocationInputPlugin"]], "datafusion.input.location.LocationInputPlugin": [[8, 2, 1, "", "build_table"], [8, 2, 1, "", "is_correct_input"]], "datafusion.io": [[9, 6, 1, "", "read_avro"], [9, 6, 1, "", "read_csv"], [9, 6, 1, "", "read_json"], [9, 6, 1, "", "read_parquet"]], "datafusion.object_store": [[10, 4, 1, "", "AmazonS3"], [10, 4, 1, "", "GoogleCloud"], [10, 4, 1, "", "Http"], [10, 4, 1, "", "LocalFileSystem"], [10, 4, 1, "", "MicrosoftAzure"]], "datafusion.plan": [[11, 1, 1, "", "ExecutionPlan"], [11, 1, 1, "", "LogicalPlan"]], "datafusion.plan.ExecutionPlan": [[11, 2, 1, "", "__repr__"], [11, 3, 1, "", "_raw_plan"], [11, 2, 1, "", "children"], [11, 2, 1, "", "display"], [11, 2, 1, "", "display_indent"], [11, 2, 1, "", "from_proto"], [11, 5, 1, "", "partition_count"], [11, 2, 1, "", "to_proto"]], "datafusion.plan.LogicalPlan": [[11, 2, 1, "", "__repr__"], [11, 3, 1, "", "_raw_plan"], [11, 2, 1, "", "display"], [11, 2, 1, "", "display_graphviz"], [11, 2, 1, "", "display_indent"], [11, 2, 1, "", "display_indent_schema"], [11, 2, 1, "", "from_proto"], [11, 2, 1, "", "inputs"], [11, 2, 1, "", "to_proto"], [11, 2, 1, "", "to_variant"]], "datafusion.record_batch": [[12, 1, 1, "", "RecordBatch"], [12, 1, 1, "", "RecordBatchStream"]], "datafusion.record_batch.RecordBatch": [[12, 3, 1, "", "record_batch"], [12, 2, 1, "", "to_pyarrow"]], "datafusion.record_batch.RecordBatchStream": [[12, 2, 1, "", "__aiter__"], [12, 2, 1, "", "__anext__"], [12, 2, 1, "", "__iter__"], [12, 2, 1, "", "__next__"], [12, 2, 1, "", "next"], [12, 3, 1, "", "rbs"]], "datafusion.substrait": [[13, 1, 1, "", "Consumer"], [13, 1, 1, "", "Plan"], [13, 1, 1, "", "Producer"], [13, 1, 1, "", "Serde"]], "datafusion.substrait.Consumer": [[13, 2, 1, "", "from_substrait_plan"]], "datafusion.substrait.Plan": [[13, 2, 1, "", "encode"], [13, 3, 1, "", "plan_internal"]], "datafusion.substrait.Producer": [[13, 2, 1, "", "to_substrait_plan"]], "datafusion.substrait.Serde": [[13, 2, 1, "", "deserialize"], [13, 2, 1, "", "deserialize_bytes"], [13, 2, 1, "", "serialize"], [13, 2, 1, "", "serialize_bytes"], [13, 2, 1, "", "serialize_to_plan"]], "datafusion.udf": [[14, 1, 1, "", "Accumulator"], [14, 1, 1, "", "AggregateUDF"], [14, 1, 1, "", "ScalarUDF"], [14, 1, 1, "", "Volatility"], [14, 1, 1, "", "WindowEvaluator"], [14, 1, 1, "", "WindowUDF"], [14, 4, 1, "", "_R"], [14, 4, 1, "", "udaf"], [14, 4, 1, "", "udf"], [14, 4, 1, "", "udwf"]], "datafusion.udf.Accumulator": [[14, 2, 1, "", "evaluate"], [14, 2, 1, "", "merge"], [14, 2, 1, "", "state"], [14, 2, 1, "", "update"]], "datafusion.udf.AggregateUDF": [[14, 2, 1, "", "__call__"], [14, 3, 1, "", "_udaf"], [14, 2, 1, "", "udaf"]], "datafusion.udf.ScalarUDF": [[14, 2, 1, "", "__call__"], [14, 3, 1, "", "_udf"], [14, 2, 1, "", "udf"]], "datafusion.udf.Volatility": [[14, 3, 1, "", "Immutable"], [14, 3, 1, "", "Stable"], [14, 3, 1, "", "Volatile"], [14, 2, 1, "", "__str__"]], "datafusion.udf.WindowEvaluator": [[14, 2, 1, "", "evaluate"], [14, 2, 1, "", "evaluate_all"], [14, 2, 1, "", "evaluate_all_with_rank"], [14, 2, 1, "", "get_range"], [14, 2, 1, "", "include_rank"], [14, 2, 1, "", "is_causal"], [14, 2, 1, "", "memoize"], [14, 2, 1, "", "supports_bounded_execution"], [14, 2, 1, "", "uses_window_frame"]], "datafusion.udf.WindowUDF": [[14, 2, 1, "", "__call__"], [14, 2, 1, "", "_create_window_udf"], [14, 2, 1, "", "_create_window_udf_decorator"], [14, 2, 1, "", "_get_default_name"], [14, 2, 1, "", "_normalize_input_types"], [14, 3, 1, "", "_udwf"], [14, 2, 1, "", "udwf"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "data", "Python data"], "5": ["py", "property", "Python property"], "6": ["py", "function", "Python function"]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:attribute", "4": "py:data", "5": "py:property", "6": "py:function"}, "terms": {"": [1, 2, 3, 4, 5, 14, 16, 17, 22, 25, 27, 28, 29, 30], "0": [2, 3, 4, 5, 14, 19, 20, 21, 22, 23, 26, 27, 29, 31, 32, 39, 40], "00": 23, "01": [19, 23, 26], "01t00": 23, "03": 23, "04023": 4, "05": [23, 26], "06": [23, 26], "08": 23, "09": [19, 23], "1": [2, 3, 4, 5, 14, 15, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 40], "10": [4, 5, 14, 18, 19, 20, 22, 23, 26, 27, 28, 29, 31, 32, 40], "100": [4, 18, 19, 20, 29, 31, 32], "1000": [1, 5, 9, 21], "10000000": 30, "1024": [1, 5], "103": [18, 31, 32], "104": [18, 31, 32], "105": [18, 20, 31, 32], "107": 20, "109": [18, 31, 32, 40], "11": [18, 19, 20, 22, 23, 27, 29, 31, 32], "110": 20, "111": [18, 31, 32], "112": 20, "115": [18, 20, 31, 32], "12": [17, 18, 19, 20, 22, 23, 27, 29, 31, 32], "120": [4, 18, 20, 31, 32], "121": 20, "122": [18, 31, 32], "123": [18, 21, 31, 32], "12371": 4, "125": [4, 20, 23, 40], "128": 4, "13": [18, 20, 22, 23, 27, 29, 31, 32], "130": [18, 31, 32], "135": [18, 31, 32], "136": 20, "14": [18, 19, 20, 22, 23, 26, 27, 31, 32], "140": 20, "14097612086726": 21, "145": [18, 20, 31, 32], "149": 40, "15": [18, 19, 20, 22, 23, 26, 27, 29, 31, 32], "150": [18, 29, 31, 32], "158": 40, "159": [18, 31, 32, 40], "16": [19, 20, 22, 23, 27], "160": 40, "161": 40, "162": 40, "163": 40, "165": [23, 40], "17": [19, 20, 22, 23, 27, 29], "18": [19, 20, 22, 23, 27, 29], "19": [20, 22, 23, 40], "190": 40, "1902": 4, "1921": 23, "195": [18, 31, 32], "1970": 23, "2": [2, 3, 4, 5, 14, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 40], "20": [2, 4, 5, 14, 18, 20, 22, 23, 28, 29, 31, 32, 40], "200": 40, "2025": 23, "205": [18, 31, 32], "21": [19, 20, 22, 23, 29], "22": [2, 22, 31], "223": 40, "224": 4, "229": 40, "23": [19, 22, 29, 40], "23076923076923": 20, "2345": 4, "24": [19, 22], "25": [4, 18, 20, 22, 23, 29, 31, 32], "256": 4, "25806451612904": 20, "26": [22, 29], "27": 22, "28": [19, 22, 29], "28571428571429": 20, "29": 22, "3": [2, 4, 14, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 40], "30": [18, 20, 22, 28, 29, 31, 32], "309": [18, 23, 31, 32], "30t17": 23, "31": 22, "314": [18, 23, 31, 32], "318": [18, 23, 31, 32], "32": 22, "33": [19, 22], "333333333333332": 29, "333333333333336": 20, "34": [22, 23], "35": [18, 19, 20, 22, 26, 29, 31, 32], "36": [19, 22], "37": 22, "38": 22, "384": 4, "39": [18, 22, 31, 32, 40], "395": [18, 31, 32], "3rd": 4, "4": [2, 4, 14, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 40], "40": [4, 17, 18, 20, 22, 28, 31, 32], "405": [18, 23, 31, 32], "41": 22, "42": [20, 22, 29, 31], "42857142857143": 29, "43": [18, 22, 29, 31, 32, 39], "44": [18, 22, 31, 32], "45": [18, 20, 22, 23, 29, 31, 32], "4579": 23, "46": [20, 32], "47": 20, "4732": 23, "48": [18, 20, 31, 32], "49": [18, 23, 31, 32], "495": [18, 31, 32], "5": [2, 4, 14, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 40], "50": [4, 18, 20, 28, 31, 32], "51": [19, 20], "512": 4, "5129": 23, "52": [18, 20, 31, 32], "525": [18, 23, 31, 32], "53": 20, "530": [18, 31, 32], "531": 21, "534": [18, 23, 31, 32], "536576280z": 23, "54": [20, 22, 23], "55": [18, 20, 29, 31, 32], "56": 20, "58": [18, 20, 31, 32], "5811388300841898": 21, "59": [18, 31, 32], "6": [18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 31, 32, 33, 40], "60": [4, 18, 20, 29, 31, 32], "62": [18, 31, 32], "625": [18, 23, 31, 32], "63": [18, 20, 31, 32], "630": [18, 31, 32], "634": [18, 23, 31, 32], "64": [17, 18, 29, 31, 32], "65": [18, 20, 22, 26, 29, 31, 32], "66": [19, 20], "666666666666668": 29, "66666666666667": 20, "666667": 4, "669": 21, "67": [20, 22], "7": [18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 31, 32, 33, 40], "70": [18, 20, 29, 31, 32], "71": [20, 22], "72": 20, "721": 21, "73": 19, "732": 23, "75": [4, 18, 20, 31, 32], "76": 29, "774": 21, "77777777777777": 20, "78": [18, 19, 31, 32], "785714285714285": 20, "78571428571429": 20, "79": [18, 19, 31, 32], "8": [18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 30, 31, 32, 33], "80": [4, 18, 20, 29, 31, 32], "803": 21, "81": 19, "82": [18, 31, 32], "83": [18, 20, 31, 32], "832": 21, "833333333333336": 20, "84": [18, 29, 31, 32], "85": [18, 20, 31, 32], "855": 23, "86": 20, "86740bfd3d9831d6b7c1d0e1bf4a21d91598a0ac": [3, 5], "88888888888889": 20, "9": [4, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 31, 32, 33, 40], "90": [4, 18, 20, 29, 31, 32], "91": 20, "92": 19, "94": [19, 26], "95": [19, 20, 29, 40], "96": [19, 20], "972": 23, "98": [20, 40], "A": [1, 2, 3, 4, 5, 6, 11, 13, 14, 19, 21, 25, 29, 39], "AND": [2, 3, 4, 5, 14], "As": [4, 5, 14, 16, 22, 26, 31], "At": [4, 16], "BY": 14, "Be": 2, "By": [2, 16, 17, 20], "For": [1, 2, 4, 13, 14, 16, 17, 20, 22, 23, 25, 26, 27, 32], "INTO": [1, 5], "If": [1, 2, 3, 4, 5, 6, 9, 14, 16, 17, 20, 23, 25, 27, 29, 32, 39], "In": [5, 14, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 31, 32, 33, 40], "It": [1, 2, 3, 4, 5, 14, 16, 18, 19, 20, 22, 25, 33, 38], "Its": 18, "No": 2, "Not": [3, 5], "OR": [3, 4, 5], "On": [2, 16, 31], "One": [2, 3, 16, 20, 29], "That": [3, 5, 33], "The": [1, 2, 4, 5, 7, 8, 9, 11, 13, 14, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 29, 31], "Then": 27, "There": [4, 23, 27], "These": [4, 5, 12, 14, 20, 22, 30], "To": [2, 4, 16, 17, 22, 23, 26, 27, 29, 31, 32, 33, 39], "With": [3, 5, 32, 33], "__add__": [3, 5], "__aiter__": [5, 12], "__and__": [3, 5], "__anext__": [5, 12], "__arrow_c_array__": [1, 33], "__arrow_c_stream__": [1, 2, 33], "__call__": [5, 14], "__datafusion_table_provider__": [1, 16, 39], "__eq__": [3, 5], "__ge__": [3, 5], "__getitem__": [2, 3, 5], "__gt__": [3, 5], "__init__": [5, 14, 27], "__invert__": [3, 5], "__iter__": [5, 12], "__le__": [3, 5], "__lt__": [3, 5], "__mod__": [3, 5], "__mul__": [3, 5], "__ne__": [3, 5], "__next__": [5, 12], "__or__": [3, 5], "__radd__": [3, 5], "__rand__": [3, 5], "__repr__": [2, 3, 5, 11], "__richcmp__": [3, 5], "__rmod__": [3, 5], "__rmul__": [3, 5], "__ror__": [3, 5], "__rsub__": [3, 5], "__rtruediv__": [3, 5], "__str__": 14, "__sub__": [3, 5], "__truediv__": [3, 5], "__version__": 32, "_aggreg": 4, "_create_window_udf": [5, 14], "_create_window_udf_decor": [5, 14], "_get_default_nam": [5, 14], "_intern": [0, 2, 3, 5, 11, 12, 13], "_normalize_input_typ": [5, 14], "_null_treat": 3, "_order_bi": 3, "_partition_bi": 3, "_r": [5, 14], "_raw_plan": [5, 11], "_repr_html_": 2, "_sum": [5, 14, 27], "_to_pyarrow_typ": [3, 5], "_udaf": [5, 14], "_udf": [5, 14], "_udwf": [5, 14], "_window_fram": 3, "_window_funct": 4, "a0": 22, "a_siz": 22, "ab": 4, "abc": 6, "abi": 16, "abi3": 17, "abi_st": 16, "abil": 31, "abilti": 31, "abl": [5, 11], "about": [13, 16, 18, 29, 30], "abov": [4, 16, 20, 29], "absolut": 4, "abstract": [5, 6, 14, 22, 27], "accept": [1, 3, 5, 33], "access": [5, 9, 16, 22, 31], "access_key_id": 31, "account": 31, "accum": [5, 14], "accumul": [5, 14, 27], "achiev": 18, "aco": 4, "acosh": 4, "acronym": 16, "across": [14, 16, 22], "act": [5, 11], "activ": [16, 17], "actual": 2, "ad": 2, "adapt": 16, "add": [1, 2, 3, 4, 16, 17], "add_3": 2, "addit": [2, 3, 4, 5, 13, 14, 16, 17, 27, 31], "addition": [16, 19], "adhoc": 17, "adopt": 16, "advanc": 31, "advantag": [16, 17, 18], "affect": [14, 20], "after": [2, 4, 14, 27], "ag": 22, "against": [1, 3, 4, 14, 16, 18, 28], "age_col": 22, "age_in_year": 22, "agg": 2, "aggreg": [1, 2, 3, 4, 5, 14, 19, 24], "aggregatefunct": 3, "aggregateudf": [1, 5, 14], "aim": 32, "albert": 22, "algorithm": [2, 4], "alia": [1, 2, 3, 4, 5, 19, 20, 22, 23, 26, 27, 29, 33], "alias": 17, "alic": 25, "all": [0, 1, 2, 4, 5, 14, 16, 17, 19, 20, 23, 25, 26, 27, 29, 31], "alloc": 2, "allow": [1, 5, 14, 16, 17, 18, 20, 22, 23, 27], "alpha": [27, 31], "alreadi": [4, 16], "also": [1, 2, 5, 14, 16, 17, 18, 20, 22, 23, 27, 29, 32, 40], "altern": [26, 35, 38], "alternate_a": 2, "alwai": 14, "amazons3": [10, 31], "amount": [2, 14], "an": [1, 2, 3, 4, 5, 9, 11, 13, 14, 16, 17, 19, 20, 21, 22, 25, 26, 27, 29, 31, 32, 33, 34, 35, 38], "analyt": 29, "analyz": [2, 3, 17], "ani": [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 16, 20, 25, 27, 29, 31, 33], "anoth": [2, 4, 5, 9, 16], "anti": [2, 24], "apach": [1, 3, 4, 5, 16, 17, 18, 33], "apart": 18, "api": [1, 5, 11, 16, 17, 18, 23, 29, 30, 40], "appear": [3, 4, 5, 16], "append": [4, 27], "appli": 2, "applic": 16, "approach": [4, 20, 22], "appropri": [14, 17, 31], "approx_distinct": [4, 20], "approx_median": [4, 20], "approx_percentile_cont": [4, 20], "approx_percentile_cont_with_weight": [4, 20], "approxim": [4, 20], "ar": [1, 2, 3, 4, 5, 9, 11, 12, 14, 16, 17, 19, 20, 22, 24, 26, 27, 29, 31, 32, 33], "arbitrari": [2, 4], "arc": [4, 16, 39], "area": 16, "arg": [1, 2, 4, 5, 13, 14, 17], "argument": [2, 4, 5, 14, 20, 27], "around": [4, 17], "arrai": [1, 2, 3, 4, 5, 14, 20, 24, 27, 31, 33], "array1": 4, "array2": 4, "array_agg": [4, 20], "array_append": [4, 17], "array_cat": [4, 22], "array_concat": [4, 22], "array_dim": 4, "array_distinct": 4, "array_el": [3, 4, 5, 22], "array_empti": [4, 22], "array_except": 4, "array_extract": 4, "array_ha": 4, "array_has_al": 4, "array_has_ani": 4, "array_indexof": 4, "array_intersect": 4, "array_join": 4, "array_length": 4, "array_ndim": 4, "array_pop_back": 4, "array_pop_front": 4, "array_posit": 4, "array_prepend": 4, "array_push_back": 4, "array_push_front": 4, "array_remov": 4, "array_remove_al": 4, "array_remove_n": 4, "array_repeat": [4, 22], "array_replac": 4, "array_replace_al": 4, "array_replace_n": 4, "array_res": 4, "array_slic": 4, "array_sort": 4, "array_to_str": 4, "array_union": 4, "arrow": [1, 2, 3, 4, 5, 17, 18, 21, 31, 32, 36], "arrow_cast": [3, 4, 5, 23], "arrow_datafusion_python_root": 17, "arrow_typeof": 4, "arrowarrayexport": 1, "arrowstreamexport": 1, "arxiv": 4, "as_pi": [5, 14, 27], "ascend": [3, 4, 5, 20, 29], "ascii": 4, "asin": 4, "asinh": 4, "assembl": 1, "assign": [3, 4, 5], "assist": 17, "associ": [0, 1, 3, 4, 5, 16], "assum": [2, 4, 17, 31], "assumpt": 18, "async": [5, 12], "atan": 4, "atan2": 4, "atanh": 4, "atk": [18, 31, 32], "attack": [18, 20, 23, 29, 31, 32, 40], "attempt": [1, 2, 5, 14, 16, 17], "auto": 15, "autoapi": 15, "automat": [1, 5], "avail": [2, 20, 24, 27, 30, 39], "availbl": 29, "averag": [4, 14, 29], "avg": [4, 14, 20, 29], "avoid": [1, 5, 9], "avro": [1, 5, 9, 31, 36], "awar": [2, 26], "aws_access_key_id": 31, "aws_secret_access_kei": 31, "b": [2, 4, 21, 22, 27, 28, 31, 33], "back": [3, 16, 18, 33], "bar": 30, "base": [1, 2, 3, 4, 5, 7, 8, 9, 14, 15, 22, 25, 27], "base64": 4, "baseinputsourc": [6, 7, 8], "basi": [5, 14, 27], "basic": [19, 24, 26, 27], "basic_typ": 23, "batch": [1, 2, 5, 11, 12, 14, 21, 27, 28, 31, 33], "batch_arrai": 27, "batch_siz": [1, 5], "becaus": [3, 5, 27], "becca": 22, "beedril": [18, 29, 31, 32], "beedrillmega": [18, 29, 31, 32], "been": 16, "befor": [4, 17, 22, 27], "beforehand": [1, 5], "begin": [1, 3, 4, 5, 14], "beginnig": 4, "being": 2, "below": [19, 20, 31], "best": [1, 5, 16], "beta": 31, "better": 2, "between": [1, 3, 4, 5, 14, 16, 18, 19, 25, 27, 29], "betwen": 29, "bia": [5, 14], "bias_10": [5, 14], "biased_numb": [5, 14], "biasednumb": [5, 14], "bin": [4, 17], "binari": [3, 4, 5, 11, 16], "binaryexpr": [3, 5], "bind": [5, 18, 32], "bit": 4, "bit_and": [4, 20], "bit_length": 4, "bit_or": [4, 20], "bit_xor": [4, 20], "bitwis": [4, 22], "blake2": 4, "blake2b": 4, "blake3": 4, "blastois": [18, 31, 32], "blastoisemega": [18, 31, 32], "blob": [3, 5], "blog": 17, "blue": 22, "bob": 25, "bool": [1, 2, 3, 4, 5, 6, 7, 8, 9, 14], "bool_": 27, "bool_and": [4, 20], "bool_or": [4, 20], "boolean": [4, 24], "bootstrap": 17, "both": [2, 4, 5, 14, 16, 17, 22, 25], "bound": [3, 5, 14, 29, 39], "boundari": [14, 16, 29], "bow": 29, "box": 16, "bracket": 22, "break": 16, "bronz": 4, "brotli": 2, "btrim": 4, "bucket_nam": 31, "bug": [18, 20, 29, 31, 32], "build": [3, 4, 5, 16, 18, 19, 20, 29], "build_flag": 17, "build_tabl": [6, 7, 8], "builder": [3, 4, 20, 29], "built": [14, 16, 20, 22, 23, 27], "bulb": 23, "bulbafleur": 23, "bulbasaur": [18, 23, 29, 31, 32], "bulk": 16, "butterfre": [18, 29, 31, 32], "byte": [1, 4, 5, 11, 13], "c": [1, 2, 14, 16, 17, 18, 21, 31, 33], "cach": 2, "calcul": [2, 4, 14, 27], "call": [0, 1, 2, 3, 4, 5, 11, 12, 13, 14, 16, 19, 20, 22, 27, 33], "callabl": [2, 5, 14], "calll": 22, "can": [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 14, 16, 17, 19, 20, 22, 26, 27, 28, 29, 30, 31, 32, 33, 39], "can_retir": 22, "cannot": 4, "canon": 16, "canonical_nam": [3, 5], "capit": [4, 26], "capsul": 16, "cardin": [4, 22], "carefulli": 16, "cargo": 17, "carlo": 22, "case": [2, 3, 4, 5, 14, 16, 20, 23, 26, 27, 29, 31], "case_build": 3, "casebuild": [3, 4], "cast": [2, 3, 4, 5, 24], "castabl": [3, 5], "catalog": [1, 5, 15], "caterpi": [18, 29, 31, 32], "caus": [1, 5, 17, 29, 33], "cbrt": 4, "cdatainterfac": 1, "ceil": 4, "certain": 14, "certainli": 17, "chain": 2, "challeng": 16, "chang": [14, 17, 22], "chansei": 20, "char": 23, "char_length": [4, 23], "charact": 4, "character_length": 4, "charizard": [18, 23, 29, 31, 32], "charizardmega": [18, 23, 29, 31, 32], "charli": 25, "charmand": [18, 23, 29, 31, 32], "charmeleon": [18, 23, 29, 31, 32], "check": [4, 22, 23], "checksum": 4, "children": [5, 11], "chr": 4, "chrono": 4, "ci": [16, 17], "citycab": 25, "class": 27, "classmethod": [1, 2], "classvar": [3, 5], "claus": [14, 29], "clean": 17, "clefabl": 29, "clefairi": [20, 29], "clone": [17, 39], "close": 29, "co": 4, "coalesc": [4, 23], "code": [4, 16, 23], "codebas": 17, "coeffici": 4, "coerc": 4, "col": [2, 3, 4, 5, 14, 19, 20, 22, 23, 26, 27, 28, 29, 33, 40], "col1": 14, "col_attack": 20, "col_diff": 27, "col_spe": 20, "col_type_1": 20, "col_type_2": 20, "collect": [2, 18, 19, 21, 28, 33, 40], "collect_partit": 2, "color": 22, "column": [1, 2, 3, 4, 5, 9, 14, 19, 20, 21, 23, 24, 25, 27, 29, 40], "column_a": 3, "column_nam": [3, 5], "com": [3, 4, 5, 17], "combin": [2, 4, 22, 25, 26, 27], "come": [31, 37], "command": [1, 5, 17], "common": [3, 4, 5, 6, 7, 8, 20, 23, 25, 29, 33], "commun": [16, 17], "compar": [4, 29], "comparison": [3, 5, 20], "compat": [16, 22], "compil": [16, 22], "complet": [3, 4, 5, 6, 16, 17, 27, 39], "complex": [2, 18, 23], "complic": 2, "compon": 4, "compos": 4, "compress": [1, 2, 5, 9], "compression_level": 2, "comput": [2, 3, 4, 5, 14, 16, 22, 27], "concat": 4, "concat_w": 4, "concaten": [4, 22], "concatenated_arrai": 22, "concept": [1, 2, 3, 5, 22], "concis": 17, "concurr": [1, 5], "condit": 24, "config": [1, 5, 17, 30], "config_intern": [1, 5], "config_opt": [1, 5], "configur": [1, 5, 17], "conflict": [1, 5, 9], "conjunct": 2, "connect": [1, 19], "consecut": 4, "consid": 4, "consist": 4, "consol": 2, "construct": [3, 5, 14, 27], "constructor": [0, 2, 3, 5, 11, 12, 13, 14], "consum": [6, 13, 33], "contain": [2, 4, 5, 11, 14, 15, 16, 17, 20, 22, 25, 27], "content": [17, 24], "context": [4, 5, 9, 11, 13, 15, 28, 30, 33, 40], "continu": [4, 16], "contribut": 17, "control": [1, 5, 20, 29], "conveni": [5, 14, 31], "convent": 16, "convers": [16, 17], "convert": [1, 2, 3, 4, 5, 11, 12, 13, 14, 16, 19, 21, 22, 23, 27, 28, 40], "copi": [16, 17, 18, 27, 33], "core": [2, 3, 5, 16], "corr": [4, 20], "correctli": [2, 14], "correl": 4, "correspond": [4, 25], "cosh": 4, "cosin": 4, "cost": 18, "costli": 14, "cot": 4, "cotang": 4, "could": [2, 3, 5, 16], "count": [2, 4, 20, 21], "count_star": 4, "counterpart": [4, 16], "coupl": [16, 29], "covar": 4, "covar_pop": [4, 20], "covar_samp": [4, 20], "covari": 4, "cover": [19, 23, 30], "cpython": 17, "crate": 16, "creat": [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 15, 16, 17, 19, 20, 22, 28, 30, 33, 40], "create_datafram": [1, 27, 31], "create_dataframe_from_logical_plan": 1, "creatememoryt": 3, "createview": 3, "credenti": 31, "criteria": [4, 29], "cstream": 16, "cstring": [16, 39], "csv": [1, 2, 5, 9, 16, 18, 20, 23, 29, 31, 32, 36, 40], "ctx": [1, 4, 5, 11, 13, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40], "cube": 4, "cubon": 20, "cume_dist": [4, 14, 29], "cumul": 4, "curr_valu": 27, "current": [2, 3, 4, 5, 11, 14, 16, 23, 27, 29], "current_d": 4, "current_tim": 4, "custom": [1, 5, 6, 11, 16, 25, 27, 36], "customer_id": 25, "d": [2, 14, 33], "dai": [4, 23], "dant": 22, "dark": 20, "data": [0, 1, 2, 3, 4, 5, 9, 11, 14, 16, 17, 18, 19, 20, 21, 22, 23, 26, 27, 28, 29, 32, 33, 34, 37, 39], "data_typ": 4, "databas": [0, 5, 22], "dataflow": [5, 11], "datafram": [1, 3, 4, 5, 9, 11, 12, 14, 15, 18, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 40], "datafrm": 2, "datafus": [15, 16, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40], "datafusion_df": 31, "datafusion_table_provid": [16, 39], "dataset": [1, 23, 29, 31, 32], "datasourc": 6, "datastructur": [3, 5], "datatyp": [2, 3, 5, 14], "datatypemap": [3, 5], "date": [4, 23], "date32": 4, "date_bin": 4, "date_part": [4, 23], "date_trunc": 4, "datepart": 4, "datetrunc": 4, "db": [0, 5], "ddl": [1, 5], "decid": [14, 16], "decim": 4, "decimal_plac": 4, "decod": 4, "decor": [5, 14], "def": [2, 5, 14, 18, 27, 31, 32], "default": [1, 2, 3, 4, 5, 8, 9, 14, 16, 17, 27, 29, 30], "default_valu": 4, "defens": [18, 23, 31, 32, 40], "defin": [1, 3, 5, 14, 16, 17, 23, 24, 29], "definit": [1, 3, 5, 14, 16, 19, 27], "degre": 4, "delet": [1, 5], "delimit": [1, 4, 5, 9], "delta": 16, "delta_t": 31, "deltalak": 31, "deltat": 31, "demonstr": [1, 27, 29, 32, 33], "dens": 4, "dense_rank": [4, 14, 29], "depend": [4, 16, 20, 27, 29], "deprec": [2, 4, 29], "deregister_t": 1, "descend": 4, "describ": [2, 4, 16, 19, 20, 21, 31], "descript": [4, 5, 14], "deseri": 13, "deserialize_byt": 13, "design": [16, 24], "desir": 4, "detafram": 1, "detail": [2, 3, 4, 5, 14, 17, 21], "determin": [2, 3, 4, 5, 27], "dev": 17, "develop": 16, "deviat": 4, "df": [1, 2, 3, 4, 18, 19, 20, 21, 22, 23, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 40], "df_filter": 28, "df_view": 28, "dfschema": 5, "dict": [1, 2, 3, 5], "dictionari": [1, 2, 22, 28, 31], "differ": [2, 3, 4, 5, 11, 14, 16, 23, 27, 31], "difficult": 16, "digest": 4, "dimens": 4, "dimension": 2, "directli": [3, 4, 13, 31, 33], "directori": [1, 5, 17], "disabl": [1, 5], "discard": 2, "discuss": 16, "disk": [1, 5, 7, 8], "displai": [5, 11, 19, 21, 28, 32], "display_graphviz": [5, 11], "display_ind": [5, 11], "display_indent_schema": [5, 11], "display_nam": [3, 5], "distanc": 4, "distinct": [2, 3, 4, 5], "distinctli": 16, "distinguish": 16, "distribut": 4, "divid": 4, "divis": [3, 4, 5], "divisor": 4, "dml": [1, 5], "do": [1, 5, 14, 16, 17, 19, 20, 27, 29, 31, 39], "doc": [1, 4], "docstr": 17, "document": [1, 2, 3, 4, 5, 15, 16, 17, 19, 30, 32], "doe": [2, 4, 14, 16, 17, 29, 31], "doesn": 14, "dominant_typ": 23, "done": [2, 20, 27], "dot": [5, 11], "doubl": [26, 33], "double_func": [5, 14], "double_it": [5, 14], "double_udf": [5, 14], "down": [16, 29, 31], "downcast": 16, "download": [26, 32], "dragon": [18, 20, 23, 29, 31, 32], "dragonair": 29, "dratini": 29, "drop": [1, 2, 5], "droptabl": 3, "dtype": 2, "due": [1, 5, 9, 17], "duplic": [2, 4, 25], "dure": [5, 14, 17], "dynamic_lookup": 17, "e": [2, 4, 14, 27, 39], "each": [2, 3, 4, 5, 14, 16, 17, 20, 29], "earli": 16, "easi": [16, 37], "easier": [16, 23, 32], "easili": [3, 5, 16], "east": 31, "effect": [26, 32], "effort": 16, "either": [1, 2, 3, 5, 14, 16, 20], "electr": [20, 29], "element": [2, 3, 4, 5, 22], "ellipsi": [2, 5, 14], "els": 27, "else_expr": 3, "employe": 22, "empti": [1, 4, 5, 11, 20, 22], "empty_t": 1, "emptyrel": 3, "enabl": [1, 5, 16], "enable_url_t": 1, "encod": [4, 13], "encourag": 17, "end": [0, 2, 3, 4, 5, 11, 12, 14, 29], "end_bound": [3, 5], "ends_with": 4, "engin": [1, 5, 18, 19], "enough": 14, "ensur": 6, "entir": [14, 20, 27, 29], "entri": [2, 4, 20, 29], "enum": [2, 3, 5, 14], "enumer": 14, "enviorn": 17, "environ": [17, 18], "equal": [2, 3, 4, 5], "equival": [4, 14, 16, 20, 29], "error": [1, 3, 5, 17], "especi": 17, "essenti": [5, 12, 21], "etc": [1, 3, 5, 7, 8, 14], "eval_rang": 14, "evalu": [2, 3, 4, 5, 14, 19, 20, 22, 27, 29], "evaluate_al": [5, 14, 27], "evaluate_all_with_rank": [14, 27], "even": [1, 5, 16, 25], "everi": [4, 5, 14], "everyth": [7, 8], "ex": [3, 5, 26], "exact": 16, "exactli": [2, 29], "examin": [3, 5], "exampl": [1, 2, 3, 4, 5, 11, 14, 16, 19, 20, 22, 23, 25, 27, 28, 29, 31, 32, 39], "excel": 16, "except": [2, 3, 4, 5, 22], "except_al": 2, "exclud": [2, 25], "execut": [1, 2, 5, 14, 17, 18, 19, 30, 33], "execute_stream": [2, 5, 12], "execute_stream_partit": 2, "execution_plan": 2, "executionplan": [1, 2, 5, 11], "exeggcut": 20, "exist": [1, 2, 3, 4, 5, 16], "exp": 4, "exp_smooth": 27, "expand": 2, "expect": [3, 4, 5, 14, 16, 22, 27, 29], "expens": 2, "experi": 17, "explain": [2, 3], "explan": [2, 19], "explicit": 30, "explicitli": [5, 14], "expon": 4, "exponenti": 4, "exponentialsmooth": 27, "export": [1, 2, 16, 17, 36], "expos": [1, 16, 31, 39], "expr": [1, 2, 4, 5, 9, 14, 15, 29], "expr1": 4, "expr2": 4, "express": [2, 3, 4, 5, 14, 20, 23, 24, 27, 29], "exprfuncbuild": [3, 5], "extend": [4, 6, 16], "extens": [1, 3, 5, 9], "extract": [3, 4, 5, 23], "extraenv": 17, "f": [3, 19, 20, 22, 23, 29, 31], "face": 16, "fact": [16, 27], "factori": [4, 5, 14], "fail": 17, "fair": [1, 5], "fairi": [20, 29], "fals": [2, 3, 4, 5, 14, 16, 17, 18, 22, 23, 26, 27, 30, 31, 32, 39], "familiar": 17, "far": 2, "faster": [4, 14], "featur": [4, 16, 17, 31, 33], "fetch": 17, "few": [16, 17, 19], "ffi": 31, "ffi_": 16, "ffi_provid": 16, "ffi_tableprovid": [1, 16, 39], "field": [2, 4, 23], "fight": [20, 29], "file": [1, 2, 5, 7, 8, 9, 13, 16, 17, 18, 19, 26, 32, 34, 35, 37, 38], "file_compression_typ": [1, 5, 9], "file_extens": [1, 5, 9], "file_partition_col": [1, 5, 9], "file_sort_ord": [1, 5, 9], "filenam": 17, "fill": [3, 4, 5, 29], "fill_nan": [3, 5], "fill_nul": [3, 5], "filter": [2, 3, 4, 5, 11, 19, 22, 28, 29, 31], "final": [19, 27], "find": [4, 16, 17, 19, 20, 29], "find_in_set": 4, "finish": [3, 20], "fire": [18, 20, 23, 31, 32], "first": [1, 2, 3, 4, 5, 16, 17, 19, 20, 22, 29, 32], "first_1": 20, "first_2": 20, "first_arrai": 4, "first_valu": [4, 14, 20], "fix": 14, "flag": [4, 14, 17], "flatten": 4, "fleur": 23, "float": [3, 4, 5, 14, 27], "float64": [5, 14, 23, 27], "floor": 4, "flow": [5, 11], "flower": 23, "fly": [18, 20, 29, 31, 32], "fn": 39, "focus": 16, "folder": [16, 39], "follow": [1, 2, 3, 4, 5, 14, 16, 17, 19, 20, 22, 23, 25, 27, 29, 31, 32], "foo": 30, "fora": [3, 5], "foreign": 16, "foreign_provid": 16, "foreigntableprovid": 16, "form": [2, 4, 5, 11, 20], "format": [1, 2, 4, 5, 11, 31, 34, 37], "format_argu": 4, "formatt": 4, "forth": 33, "found": [4, 17, 29, 39], "four": 27, "frame": [3, 4, 5, 14, 21, 27], "frame_bound": 3, "framework": 17, "free": [1, 5], "frequent": [16, 17], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 40], "from_arrai": [27, 31], "from_arrow": [1, 31, 33], "from_arrow_t": 1, "from_panda": 1, "from_polar": [1, 31], "from_proto": [5, 11], "from_pydict": [1, 21, 22, 25, 27, 28, 31, 33], "from_pylist": [1, 25, 31], "from_str": 2, "from_substrait_plan": 13, "from_unixtim": 4, "from_val": 4, "fuction": 14, "full": [2, 4, 24, 40], "fulli": 17, "func": [2, 5, 14], "function": [1, 2, 3, 10, 12, 14, 15, 16, 19, 21, 24, 31, 38], "function_to_impl": [14, 27], "further": 16, "futher": 4, "futur": [14, 29], "g": [4, 14], "gamma": 31, "gcd": 4, "gener": [3, 5, 11, 12, 13, 14, 15, 16, 17, 18, 31, 32], "geodud": 29, "get": [2, 5, 11, 14, 17, 22, 31], "get_default_level": 2, "get_frame_unit": [3, 5], "get_lower_bound": [3, 5], "get_offset": 3, "get_rang": 14, "get_upper_bound": [3, 5], "getenv": 31, "ghost": 20, "gil": 18, "git": 17, "github": [3, 4, 5, 17], "give": [4, 20, 26, 32], "given": [0, 1, 3, 4, 5, 14, 27], "glanc": 16, "global": [1, 5, 9], "global_ctx": 1, "go": [16, 21, 23], "goe": 16, "gold": 4, "good": [16, 17], "googlecloud": [10, 31], "graph": [5, 11], "graphic": [5, 11], "graphviz": [5, 11], "grass": [18, 20, 23, 29, 31, 32], "great": 17, "greater": [3, 4, 5], "greatest": 4, "greatli": 27, "greedi": [1, 5], "green": 22, "grimer": 29, "ground": 20, "group": [1, 2, 3, 4, 5, 9, 14, 19, 20, 21, 29], "group_bi": [2, 20], "groupingset": 3, "guarante": 16, "guid": [1, 24, 30, 32], "gzip": 2, "ha": [1, 2, 4, 5, 14, 16, 22, 26, 31], "handl": [12, 20, 29], "happen": [17, 33], "has_head": [1, 5, 9], "hash": [2, 4], "haskel": 21, "have": [1, 2, 4, 5, 9, 13, 14, 16, 17, 18, 20, 25, 27, 29, 39], "head": 2, "header": [1, 2, 5, 9], "healthi": 17, "heavy_red_unit": 22, "help": [1, 5, 9, 17, 22, 23], "helper": [5, 14], "henc": 14, "here": [4, 16, 23, 26, 29, 32, 33, 40], "hex": 4, "hexadecim": 4, "high": [3, 5], "higher": [2, 3, 5], "highli": 27, "highlight": 32, "hint": [1, 17], "homebrew": 17, "host": 1, "how": [1, 2, 3, 4, 5, 14, 16, 20, 21, 24, 25, 27, 29, 30, 32], "howev": [14, 27], "hp": [18, 31, 32], "html": [1, 4], "http": [1, 3, 4, 5, 10, 11, 13, 31], "hyperbol": 4, "i": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 16, 17, 18, 19, 20, 22, 23, 25, 27, 29, 31, 32, 33, 34, 35, 37, 38], "ic": 20, "id": [1, 17, 25], "ideal": 16, "ident": 31, "identifi": [1, 3, 5, 16, 26], "idx": [14, 27], "ignor": [4, 17, 20, 29], "ignore_nul": [20, 29], "ilik": 3, "immut": [5, 14, 27], "impl": 39, "implement": [1, 2, 14, 17, 27, 31, 33, 39], "import": [1, 3, 5, 14, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 40], "improv": [1, 5, 14, 27], "in_list": [4, 23], "includ": [1, 2, 3, 5, 16, 20, 23, 25], "include_rank": [14, 27], "inclus": [1, 3, 4, 5], "incorpor": 27, "increas": [1, 5, 20], "increment": [14, 16, 27], "indent": [5, 11], "independ": [4, 29], "index": [3, 4, 5, 14, 22], "indic": [3, 4, 22], "individu": [22, 29], "infer": [1, 5, 9], "inform": [1, 2, 3, 5, 6, 13, 16, 27], "information_schema": [1, 5], "init": 17, "initcap": 4, "initi": 4, "inlin": 14, "inlist": 3, "inner": [2, 24], "input": [1, 2, 4, 5, 9, 11, 14, 15, 22, 27], "input_item": [6, 7, 8], "input_typ": [5, 14], "inputsourc": 6, "insert": [1, 5], "instanc": [1, 4, 5, 14, 19], "instanti": [5, 14], "instead": [3, 4, 5, 16], "insubqueri": 3, "int": [1, 2, 3, 4, 5, 9, 11, 14, 27], "int32": [5, 14, 23], "int64": [5, 14, 21, 23, 27], "integ": [3, 4, 5, 22], "integr": [16, 32, 39], "intend": 16, "interact": [16, 22], "intercept": 4, "interchang": 37, "interest": 22, "interfac": [1, 4, 16, 18, 19, 27, 29, 31, 33, 39], "intermedi": [5, 14], "intern": 27, "intersect": [2, 4], "interv": 4, "into_view": 2, "intro": 17, "introduc": [17, 19], "introduct": 19, "intuit": 16, "invers": 4, "invoc": 14, "io": [1, 5, 13, 15], "is_caus": 14, "is_correct_input": [6, 7, 8], "is_current_row": 3, "is_empti": 22, "is_follow": 3, "is_not_nul": [3, 5, 20], "is_nul": [3, 5, 27], "is_null_arr": 27, "is_preced": 3, "is_unbound": 3, "isfals": 3, "isnan": 4, "isnotfals": 3, "isnotnul": [3, 5], "isnottru": 3, "isnotunknown": 3, "isnul": 3, "issu": [4, 17, 29], "istru": 3, "isunknown": 3, "iszero": 4, "iter": [2, 5, 12], "iterat": 2, "its": [2, 4, 5, 11, 14, 16, 18, 29], "itself": 22, "ivi": 23, "ivyfleur": 23, "ivysaur": [18, 23, 29, 31, 32], "java": 21, "javascript": 37, "jigglypuff": 29, "join": [1, 2, 3, 5, 11, 24], "join_kei": [2, 25], "join_on": 2, "joinconstraint": 3, "jointyp": 3, "json": [1, 2, 5, 9, 31, 36], "jupyt": 32, "jupyterlab": 32, "jynx": 20, "kakuna": [18, 29, 31, 32], "keep": [17, 28], "kei": [1, 2, 3, 5, 16, 19, 22, 25], "kind": [0, 5, 16, 17], "know": [1, 5], "known": 22, "kwarg": [6, 7, 8], "kwd": [2, 14], "l179": [3, 5], "lab": 32, "label": 2, "lack": 18, "lag": [4, 14, 29], "lambda": [5, 14], "languag": [1, 5, 11, 16], "larg": [2, 23], "large_trip_dist": 26, "last": [4, 29], "last_valu": [4, 20, 29], "last_with_nul": 29, "last_wo_nul": 29, "later": [31, 32, 39], "latest": [4, 16], "latter": 1, "lazi": 19, "lcm": 4, "lead": [4, 14, 16, 27, 29, 31], "leaf": [5, 11], "leak": 18, "learn": [16, 21, 29], "least": [4, 14, 20, 25, 27], "leav": [5, 11], "left": [2, 4, 23, 24], "left_on": [2, 25], "leftmost": 4, "legendari": [18, 31, 32], "len": [14, 23], "length": [4, 5, 14], "less": [3, 4, 5], "lesson": 16, "let": [16, 30, 39], "letter": [4, 26], "level": [2, 4, 16], "levenshtein": 4, "leverag": 16, "lib": 17, "lib_dir": 17, "lib_nam": 17, "librari": [5, 6, 16, 18, 32, 33, 39], "lightweight": 37, "like": [1, 3, 4, 5, 6, 16, 17, 18, 23], "limit": [2, 3, 4, 17, 19, 21, 23], "line": [1, 2, 5, 9, 11, 16], "linear": [4, 20], "link": [17, 29], "lint": 17, "linter": 17, "list": [0, 1, 2, 3, 4, 5, 9, 11, 14, 20, 27, 28, 29, 31], "list_append": [4, 17], "list_cat": 4, "list_concat": 4, "list_dim": 4, "list_distinct": 4, "list_el": 4, "list_except": 4, "list_extract": 4, "list_indexof": 4, "list_intersect": 4, "list_join": 4, "list_length": 4, "list_ndim": 4, "list_posit": 4, "list_prepend": 4, "list_push_back": 4, "list_push_front": 4, "list_remov": 4, "list_remove_al": 4, "list_remove_n": 4, "list_repeat": 4, "list_replac": 4, "list_replace_al": 4, "list_replace_n": 4, "list_res": 4, "list_slic": 4, "list_sort": 4, "list_to_str": 4, "list_union": 4, "listingt": 1, "lit": [2, 3, 4, 5, 17, 19, 20, 22, 26, 29, 33], "liter": [2, 3, 5, 23, 24, 26, 28], "ll": [23, 25, 29], "ln": 4, "local": [1, 16, 17], "localfilesystem": [10, 31], "locat": [1, 7, 15], "locationinputplugin": [7, 8], "lock": 18, "log": [4, 23], "log10": 4, "log2": 4, "logarithm": 4, "logic": [1, 2, 3, 5, 11, 13, 19, 20, 26], "logical_plan": [2, 13], "logicalplan": [1, 2, 3, 5, 11, 13], "lonely_trip": 26, "long": 16, "long_tim": 22, "longer": 4, "look": [17, 30], "low": [3, 5, 20], "low_passenger_count": 26, "lower": [3, 4, 5, 14, 23, 26], "lowercas": [2, 4], "lowest": [16, 20], "lpad": 4, "ltrim": 4, "lz4": 2, "lz4_raw": 2, "lzo": 2, "m": 17, "mac": 17, "machin": 17, "machop": 29, "made": 26, "magikarp": 20, "magnemit": 29, "mai": [1, 2, 4, 5, 9, 14, 16, 27, 31, 32, 33], "main": [1, 19], "maintain": [1, 2, 16, 19], "major": [17, 18], "make": [4, 16, 17, 18, 23], "make_arrai": 4, "make_d": 4, "make_list": 4, "manag": [1, 5, 17], "mani": [16, 20, 27], "manipul": [1, 5, 23], "mankei": 29, "manner": 16, "manual": 17, "map": 2, "map_err": 39, "match": [2, 3, 4, 25], "math": 20, "mathemat": [24, 26], "maturin": 17, "max": [4, 20, 21], "maximum": [1, 4, 5, 9], "md5": 4, "mean": [4, 16, 17, 21], "meaning": [4, 26], "meant": [5, 11], "medal": 4, "median": [4, 20, 21], "member": 17, "memoiz": 14, "memori": [1, 2, 5, 11, 14, 18], "mention": 22, "merg": [5, 14, 25, 27], "metadata": [1, 5, 9], "metapod": [18, 20, 29, 31, 32], "method": [1, 2, 4, 5, 14, 19, 21, 25, 27, 28], "metric": 2, "metrorid": 25, "microsecond": 4, "microsoftazur": [10, 31], "might": [14, 20], "millisecond": 4, "min": [4, 20, 21], "minimum": [1, 4, 5], "model": 2, "modifi": [2, 19], "modify_df": 2, "modul": 29, "modulo": [3, 5], "moment": [2, 16], "month": [4, 23], "more": [1, 2, 3, 5, 13, 16, 17, 20, 23, 27, 30], "most": [14, 16, 22, 29, 32], "mostli": 17, "much": [14, 17], "multi": 18, "multipl": [1, 2, 3, 4, 5, 14, 20, 23, 25, 27, 29, 31], "must": [1, 2, 3, 4, 5, 14, 20, 22, 26, 27, 29, 33, 39], "my_capsul": 16, "my_delta_t": 31, "my_provid": 16, "my_tabl": [28, 39], "my_udaf": 27, "myaccumul": 27, "mytableprovid": [16, 39], "myusernam": 17, "n": [2, 4], "name": [0, 1, 2, 3, 4, 5, 9, 14, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 29, 31, 32, 39], "name_pair": 4, "named_expr": 2, "named_struct": [4, 17], "nan": [3, 4, 5], "nanosecond": 4, "nanvl": 4, "narrowli": 16, "nativ": 16, "natur": 4, "nearest": 4, "necessari": 16, "need": [1, 2, 3, 5, 9, 14, 16, 17, 27, 29, 32], "neg": [3, 4], "negat": [3, 4, 5, 23], "new": [1, 2, 3, 4, 5, 14, 16, 17, 22, 24, 39, 40], "new_bound": [16, 39], "new_err": 39, "new_nam": 2, "next": [4, 5, 12], "node": [5, 11], "non": [2, 3, 4, 20, 29], "none": [1, 2, 3, 4, 5, 9, 13, 14, 16, 27, 29, 31], "normal": 20, "not_red_unit": 22, "notat": [22, 37], "note": [1, 2, 3, 5, 19, 22, 33], "notebook": 32, "now": [4, 14, 23, 28], "nr": 21, "nth_valu": [4, 14, 20], "ntile": [4, 29], "null": [2, 3, 4, 5, 21, 25, 27, 33], "null_count": 21, "null_first": 4, "null_treat": [3, 4, 5, 20, 29], "nullabl": 2, "nullif": [4, 23], "nulls_first": [3, 4, 5], "nulltreat": [3, 4, 5, 20, 29], "num": [2, 4], "num_centroid": 4, "num_el": 22, "num_row": [5, 14, 27], "number": [1, 2, 4, 5, 9, 11, 14, 17, 22, 23, 29], "numebr": 4, "numer": [2, 4], "nvl": 4, "o": 31, "object": [1, 2, 3, 5, 10, 16, 22, 27, 30, 33, 37], "object_stor": [5, 15, 31], "objectstor": 1, "obtain": [6, 18], "obviou": 16, "occur": [5, 14, 33], "occurr": 4, "octet_length": 4, "oddish": 29, "offend": 17, "offer": [23, 40], "offici": 16, "offset": [2, 3], "often": 29, "old": 2, "old_nam": 2, "older": 31, "olymp": 4, "omit": 25, "on_expr": 2, "onc": [4, 6, 14, 39], "one": [2, 3, 4, 5, 14, 16, 20, 25, 27, 29, 31], "onli": [1, 2, 3, 4, 5, 9, 14, 16, 20, 25, 27, 29], "onlin": [1, 2, 3, 4, 5, 16, 30], "op": [2, 3, 5], "oper": [1, 2, 3, 4, 5, 11, 12, 14, 16, 18, 20, 22, 26, 27, 29, 31, 33], "operand": [3, 5], "opt": 17, "optim": [2, 14, 16], "optimized_logical_plan": 2, "option": [1, 4, 5, 9, 14, 17, 20, 23, 30, 31, 32], "options_intern": [1, 5], "order": [1, 2, 3, 4, 5, 6, 9, 14, 16, 22], "order_bi": [3, 4, 5, 20, 29], "org": [1, 4, 5, 11], "origin": [2, 4, 27], "other": [1, 2, 3, 4, 5, 14, 16, 17, 18, 22, 24], "otherwis": [3, 4], "our": [16, 17, 20, 32], "out": [2, 4, 16, 20, 21, 22, 23, 25, 26, 29, 32, 33, 40], "output": [2, 3, 5, 11, 14, 17, 20, 22, 27, 28], "over": [2, 3, 5, 14, 16, 18, 20, 29], "overhead": 14, "overlai": 4, "overlap": 4, "overrid": 27, "overridden": 20, "own": [6, 27], "pa": [1, 3, 5, 12, 14, 27, 31, 33], "packag": [4, 16, 17], "pad": 4, "page": [15, 16], "pair": 4, "panda": [1, 2, 19, 21, 31, 40], "pandas_df": 31, "para": 29, "parallel": [1, 5], "paramet": [1, 2, 3, 4, 5, 9, 13, 14, 17, 22, 24, 25], "parasect": 29, "parquet": [1, 2, 5, 9, 16, 18, 19, 26, 30, 31, 36], "parquet_prun": [1, 5, 9], "part": [4, 23], "particular": 4, "partit": [1, 2, 3, 4, 5, 9, 11, 14, 20], "partition_bi": [3, 4, 5, 29], "partition_count": [5, 11], "pass": [2, 5, 14, 16, 17, 21, 22, 27, 30, 31], "passenger_count": 26, "path": [1, 2, 5, 9, 13, 17, 31], "path_to_t": 31, "pathlib": [1, 2, 5, 9, 13], "pattern": [4, 16], "pc": [5, 14], "pcre": 4, "pd": 31, "peopl": 4, "per": [2, 14, 29], "percent": 4, "percent_rank": [4, 14, 29], "percentag": 4, "percentil": 4, "perform": [1, 2, 3, 4, 5, 14, 16, 17, 19, 20, 23, 27, 29, 31], "period": 16, "person": 17, "physic": [2, 5, 11, 19], "pi": 4, "pick": 4, "pinsir": 29, "pip": [18, 32], "pl": 31, "place": [4, 17, 32], "placehold": 3, "plan": [1, 2, 3, 5, 13, 14, 15, 18, 19], "plan_intern": 13, "planner": [5, 11], "pleas": 26, "plu": 4, "plugin": [6, 7, 8], "point": [4, 17], "pointer_width": 17, "poison": [18, 20, 29, 31, 32], "pokemon": [18, 19, 20, 23, 29, 31, 32, 40], "polar": [1, 2, 31], "polars_df": 31, "pool": [1, 5], "popul": 4, "popular": [23, 31], "portion": 16, "posit": [4, 20], "possibl": [3, 5, 14, 16, 27, 29], "post": 17, "potenti": [2, 5, 11], "pow": [4, 23], "power": [4, 23, 27, 29], "pr": 17, "practic": 16, "prece": 4, "preced": [3, 5, 14], "preceed": 29, "precis": 4, "predic": [1, 2, 5, 9], "prefer": 17, "prefix": [4, 16], "prepend": 4, "presenc": 23, "present": 25, "preserve_nul": 2, "prevent": 17, "previou": [4, 27, 29], "previous": 1, "primari": [7, 22], "print": [2, 5, 11, 28, 30], "printabl": [5, 11], "prior": [2, 31], "probabl": 20, "process": [14, 20, 29], "produc": [4, 5, 11, 13, 14, 16, 29], "product": 4, "program": 16, "programmat": [5, 11], "project": [2, 3, 5, 11, 16, 17, 33], "properti": [0, 3, 5, 11], "proto_byt": 13, "protobuf": [5, 11], "protocol": 1, "provid": [0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 16, 19, 20, 23, 27, 29, 32, 36], "prune": [1, 5, 9], "psychic": 20, "public": [0, 5], "pull": 17, "pure": 17, "push": [17, 31], "pushdown_filt": 30, "put": 26, "py": [1, 2, 5, 13, 14, 16, 39], "pyarrow": [0, 1, 2, 3, 5, 9, 12, 14, 16, 21, 27, 31, 33, 40], "pycapsul": [1, 2, 16, 31, 33, 39], "pycapsuleinterfac": 1, "pymethod": 39, "pyo3": [16, 17], "pyo3_build_config": 17, "pyo3_config_fil": 17, "pyo3_print_config": 17, "pyobject": [3, 5], "pypi": 32, "pyproject": 17, "pyresult": 39, "pyruntimeerror": 39, "pyspark": 18, "pytabl": 16, "pytest": 17, "python": [1, 3, 5, 14, 21, 22, 27, 31, 32, 33, 39], "python3": 17, "python_valu": [3, 5], "pythontyp": [3, 5], "queri": [1, 5, 11, 13, 14, 16, 18, 19, 23, 28], "quick": 21, "quit": 38, "quot": 26, "r": [2, 3, 4, 5, 16, 17], "radian": 4, "rais": [2, 4], "random": [4, 14, 21, 22], "rang": [1, 2, 3, 4, 5, 14, 21, 22, 23, 27, 29, 30], "rank": [4, 14, 27, 29], "ranks_in_partit": 14, "rather": [4, 13], "ratio": 4, "raw": 3, "raw_sort": 3, "rawexpr": [3, 5], "rb": [5, 12], "re": 16, "read": [1, 5, 7, 8, 9, 13, 16, 30, 31, 32, 34, 35, 37, 38, 40], "read_avro": [1, 5, 9, 34], "read_csv": [1, 5, 9, 18, 19, 20, 29, 31, 32, 35], "read_json": [1, 5, 9, 37], "read_parquet": [1, 5, 9, 19, 26, 38], "read_tabl": 1, "reader": [1, 5, 9, 33], "realiti": 20, "reason": 17, "rebuild": 17, "receiv": [4, 14, 16], "recent": [29, 31], "recommend": [2, 16, 17, 27], "record": [1, 2, 5, 11, 12, 26, 27, 31, 33, 34], "record_batch": [1, 2, 5, 15], "record_batch_stream": [5, 12], "recordbatch": [1, 2, 5, 12, 27, 31], "recordbatchstream": [1, 2, 5, 12], "red": 22, "red_or_green_unit": 22, "red_unit": 22, "reduc": [17, 27], "ref": 4, "refer": [2, 4, 16, 19, 23, 40], "referenc": [1, 19], "regardless": 14, "regener": 17, "regex": 4, "regexp_count": 4, "regexp_lik": 4, "regexp_match": [4, 23], "regexp_replac": [4, 23], "region": 31, "regist": [1, 5, 6, 9, 16, 19, 24, 31, 39, 40], "register_avro": 1, "register_csv": [1, 23, 35, 40], "register_dataset": [1, 31], "register_json": 1, "register_listing_t": 1, "register_object_stor": [1, 31], "register_parquet": [1, 31, 38], "register_record_batch": 1, "register_t": [1, 2], "register_table_provid": [1, 31, 39], "register_udaf": 1, "register_udf": 1, "register_udwf": 1, "register_view": [1, 28], "regr_avgi": [4, 20], "regr_avgx": [4, 20], "regr_count": [4, 20], "regr_intercept": [4, 20], "regr_r2": [4, 20], "regr_slop": [4, 20], "regr_sxi": 4, "regr_sxx": [4, 20], "regr_syi": [4, 20], "regress": [4, 20], "regular": [4, 23], "rel": [4, 29], "relat": [5, 11, 25], "releas": [16, 29], "relev": 17, "reli": 17, "remain": 4, "remot": [7, 8], "remov": [1, 2, 4, 20], "renam": 2, "renamed_ag": 22, "repartit": [1, 2, 3, 5], "repartition_by_hash": 2, "repeat": [4, 22], "repeated_arrai": 22, "replac": 4, "repo": 17, "report": [2, 17], "repositori": [16, 31], "repres": [1, 2, 3, 4, 5, 9, 11, 12, 13, 19, 22, 27], "represent": [1, 2, 3, 4, 5, 9, 11], "request": [2, 17], "requested_schema": [1, 2], "requir": [2, 3, 14, 16, 23, 31], "reserv": [1, 5], "resolv": [17, 29], "resourc": 16, "respect": [2, 4, 20, 31], "respect_nul": [4, 20, 29], "result": [1, 2, 4, 5, 11, 12, 14, 18, 19, 20, 22, 25, 26, 27, 28, 29], "result_dict": 28, "retriev": [1, 3, 5, 25], "return": [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 13, 14, 22, 25, 27, 33], "return_typ": [5, 14], "reus": 16, "reusabl": 3, "revers": 4, "review": [16, 17], "rex": [3, 5], "rex_call_oper": [3, 5], "rex_call_operand": [3, 5], "rex_typ": [3, 5], "rextyp": [3, 5], "rfc3339": 4, "rh": [3, 5], "rhyhorn": 29, "right": [2, 4, 25], "right_on": [2, 25], "ritchi": 29, "robin": 2, "rock": 20, "roll": 29, "root": [4, 5, 11], "roption": 16, "round": [2, 4, 19], "rout": 16, "row": [1, 2, 3, 4, 5, 9, 14, 19, 20, 21, 22, 25, 27, 28, 29, 40], "row_numb": [4, 14, 29], "rpad": 4, "rresult": 16, "rstring": 16, "rtrim": 4, "rubi": 21, "run": [1, 2, 5, 9, 18, 28, 32], "runnabl": 17, "runtim": [1, 5, 14, 30], "runtimeconfig": 1, "runtimeenvbuild": [1, 5, 30], "rust": [4, 16, 18, 30, 31, 39], "rustc": 16, "rustflag": 17, "rustonomicon": 16, "rvec": 16, "s3": 31, "safe": 16, "safeti": 18, "same": [2, 4, 5, 14, 16, 19, 27], "sampl": [4, 21, 28], "saur": 23, "save": 14, "scalar": [3, 5, 14, 22, 24], "scalarsubqueri": 3, "scalarudf": [1, 5, 14], "scalarvalu": 14, "scalarvari": 3, "scan": [1, 5, 11], "schema": [0, 1, 2, 3, 5, 9, 11, 21], "schema_infer_max_record": [1, 5, 9], "schema_nam": [3, 5], "scheme": 2, "scienc": 23, "score": 4, "search": 4, "second": [4, 19, 20, 22], "second_arrai": 4, "secret_access_kei": 31, "section": [16, 19, 21, 24, 25, 29, 31], "see": [1, 2, 3, 4, 5, 12, 13, 14, 16, 17, 20, 22, 27, 29], "seen": [14, 20], "select": [1, 2, 3, 4, 5, 9, 14, 19, 22, 23, 24, 27, 28, 29, 33, 40], "select_column": 2, "self": [5, 12, 14, 27, 39], "semi": [2, 24], "sensit": 2, "separ": 4, "seper": 4, "sequenc": 2, "serd": 13, "seri": 22, "serial": [13, 34], "serialize_byt": 13, "serialize_to_plan": 13, "serv": 16, "session": [1, 5, 30, 32], "session_id": 1, "sessionconfig": [1, 5, 30], "sessioncontext": [1, 2, 4, 5, 6, 11, 13, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40], "sessioncontextintern": 1, "set": [0, 1, 3, 4, 5, 14, 16, 17, 19, 24, 27, 30], "sever": 26, "sha": 4, "sha224": 4, "sha256": 4, "sha384": 4, "sha512": 4, "sharabl": 16, "share": [16, 17, 19, 22, 33], "shift_offet": 4, "shift_offset": 4, "shot": 14, "should": [1, 3, 4, 5, 6, 9, 11, 13, 17, 20, 25, 27, 29], "show": [2, 16, 18, 19, 21, 22, 25, 27, 29, 31, 32, 39], "side": [4, 16], "sign": 4, "signific": 31, "significantli": 4, "signum": 4, "silver": 4, "similar": [4, 16, 19, 22, 29], "similarto": 3, "simpl": [4, 17, 26, 37, 38], "simplest": [14, 27], "simpli": [2, 5, 14, 16, 17], "simplifi": 29, "sin": 4, "sinc": [2, 20, 27, 31], "sine": 4, "singl": [1, 2, 3, 4, 5, 11, 14, 20, 22, 27, 29], "sinh": 4, "size": [1, 2, 4, 5, 22], "skip": [1, 2, 5, 9, 14], "skip_metadata": [1, 5, 9], "slice": 4, "slope": 4, "slow": 2, "slower": 2, "slowest": 27, "slowpok": 29, "small": 31, "smallest": [14, 20], "smooth_a": 27, "snappi": 2, "snorlax": 29, "so": [1, 2, 5, 16, 17, 26, 27, 29, 33, 39], "softwar": [5, 11, 16], "some": [4, 14, 16, 17, 20, 23, 29, 32, 39], "sometim": [1, 5, 16, 31], "soon": [4, 31], "sort": [1, 2, 3, 4, 5, 9, 20, 29], "sortexpr": [1, 2, 3, 4, 5], "sound": 16, "sourc": [1, 4, 5, 6, 7, 8, 9, 16, 17, 19, 32, 33, 39], "sp": [18, 31, 32], "space": 4, "sparingli": 27, "special": 14, "specif": [3, 5, 11, 21, 22, 29], "specifi": [1, 2, 3, 4, 5, 22, 25, 27, 29], "speed": [2, 18, 20, 29, 31, 32], "sphinx": 15, "spill": [1, 5], "spillabl": [1, 5], "split": [4, 27], "split_part": 4, "sql": [1, 5, 11, 13, 14, 18, 19, 22, 26, 28], "sql_with_opt": 1, "sqloption": [1, 5], "sqltabl": [6, 7, 8], "sqltype": [3, 5], "sqrt": 4, "squar": 4, "squi": 23, "squirtl": [18, 23, 29, 31, 32], "src": [3, 5, 17], "stabl": [14, 16, 27], "standard": [4, 16], "start": [3, 4, 5, 14, 22, 26, 29, 32], "start_ag": 22, "start_bound": [3, 5], "started_young": 22, "starts_with": 4, "state": [1, 5, 14, 19, 27], "state_typ": [5, 14], "statement": [1, 3, 4, 5, 19, 40], "static": [3, 5, 11, 13, 14], "statist": [2, 20, 21], "std": 21, "stddev": [4, 20], "stddev_pop": [4, 20], "stddev_samp": 4, "steel": 20, "step": [4, 16], "still": [27, 31], "stop": 4, "storag": 31, "store": [1, 3, 5, 10], "str": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14], "str_lit": 23, "straightforward": [34, 35], "stream": [1, 2, 5, 12, 16], "strftime": 4, "stride": 4, "string": [2, 3, 4, 5, 11, 14, 16, 17, 20, 21, 22, 24], "string1": 4, "string2": 4, "string_agg": [4, 20], "string_list": 4, "string_liter": [3, 5, 23], "string_view": [3, 5, 33], "strong": 18, "strpo": 4, "struct": [1, 3, 4, 5, 16, 24, 33], "structarrai": 33, "structur": [5, 11, 16], "style": [16, 22], "sub": [3, 5], "subfield": [3, 4, 5], "submit": 17, "submodul": 17, "subqueri": 3, "subqueryalia": 3, "subset": 26, "substr": 4, "substr_index": 4, "substrait": [5, 15], "subtract": [3, 5], "suffici": [1, 5], "suffix": 4, "suggest": 31, "sum": [4, 5, 14, 20, 27], "sum_bias_10": [5, 14], "summar": [2, 5, 14], "summari": [2, 20, 21], "support": [2, 3, 4, 5, 11, 13, 16, 25, 31], "supports_bounded_execut": [14, 27], "suppos": [4, 16, 20], "suppress_build_script_link_lin": 17, "sure": 17, "symbol": 2, "sync": 17, "synchon": 16, "syntax": 4, "system": [1, 5, 16, 17], "t": [4, 14, 26], "t1": 14, "tabl": [0, 1, 2, 5, 6, 7, 8, 9, 11, 16, 19, 21, 23, 25, 32, 33, 35, 36, 38], "table_exist": 1, "table_nam": [6, 7, 8], "table_partition_col": [1, 5, 9], "table_provid": 1, "tableprovid": [16, 39], "tableproviderexport": 1, "tablescan": 3, "tail": 2, "take": [2, 14, 16, 17, 20, 22, 27, 29], "tan": 4, "tangent": 4, "tanh": 4, "target": [1, 5], "target_partit": [1, 5], "task": [23, 33], "technic": 18, "tempor": 24, "temporari": [1, 5], "term": 29, "terminologi": 16, "test": [4, 17], "text": [1, 4], "textual": 23, "th": 4, "than": [2, 3, 4, 5, 20], "thei": [6, 14, 16, 22, 27], "them": [4, 16, 17, 19, 21], "then_expr": 3, "therefor": 2, "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29, 31, 32, 33, 39], "third": 19, "thoroughli": 16, "those": [2, 3, 5, 18, 20, 27, 31], "though": 27, "thread": 18, "three": [27, 29, 31], "through": [16, 18, 24, 31, 32], "thu": 14, "thusli": 16, "ticket": 17, "tie": 4, "tight": 16, "tile": 4, "time": [4, 14, 16, 17, 20, 22, 23, 27, 33], "time64": 4, "timestamp": [4, 23], "tip_amount": [19, 26], "tip_perc": 19, "tips_plus_tol": 26, "tlc": 26, "tmp": [1, 5], "to_arrow_t": 2, "to_hex": 4, "to_panda": [2, 21, 23, 40], "to_polar": 2, "to_proto": [5, 11], "to_pyarrow": [5, 12], "to_pyarrow_dataset": 31, "to_pydict": [2, 28], "to_pylist": 2, "to_str": 39, "to_substrait_plan": 13, "to_timestamp": [4, 23], "to_timestamp_micro": 4, "to_timestamp_milli": 4, "to_timestamp_nano": 4, "to_timestamp_second": 4, "to_unixtim": 4, "to_val": 4, "to_vari": [3, 5, 11], "togeth": [2, 4, 20], "tolls_amount": 26, "toml": 17, "top": 21, "topic": 31, "total": [2, 4, 18, 19, 22, 23, 31, 32], "total_amount": 19, "total_as_float": 23, "total_as_int": 23, "toward": 4, "tracker": 4, "tradit": 17, "trait": [14, 16], "transact": [1, 5], "transfer": 16, "transform": [2, 5, 11, 19], "translat": [4, 16], "treat": [3, 29], "treatment": [3, 5], "tree": [5, 11, 22], "trigger": 2, "trim": 4, "trip": [26, 31], "trip_dist": [19, 26], "trivial": [2, 17], "tru": 2, "true": [1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 17, 20, 22, 23, 26, 27, 29, 30], "trunc": 4, "truncat": [4, 20, 23, 26, 29], "try": [1, 5, 9, 16], "trycast": 3, "tupl": [1, 2, 4, 5, 9, 14], "turn": [2, 3, 5, 16], "tutori": 32, "two": [2, 3, 4, 5, 11, 16, 20, 22, 25, 27, 29, 30], "type": [1, 2, 3, 4, 5, 9, 14, 16, 17, 18, 20, 22, 23, 26, 27, 29, 31, 32], "typic": [0, 3, 5, 12, 14, 19], "typing_extens": [5, 12], "u": [16, 20, 22, 31], "udaf": [1, 5, 14, 18, 27], "udaf1": [5, 14], "udaf2": [5, 14], "udaf3": [5, 14], "udf": [1, 5, 15, 18, 27], "udf4": [5, 14], "udwf": [1, 5, 14], "udwf1": [5, 14], "udwf2": [5, 14], "udwf3": [5, 14], "ultim": 17, "unari": [5, 11], "unbound": [1, 3, 5, 14, 29], "uncompress": 2, "undefin": 17, "under": [1, 16], "underli": [3, 5, 16], "unfortun": 16, "unicod": 4, "union": [2, 3, 4], "union_distinct": 2, "uniqu": [1, 4, 14, 20], "unit": [3, 5, 29], "unixtim": 4, "unknown": 3, "unlik": [3, 5, 29], "unmatch": 25, "unnest": [2, 3], "unnest_column": 2, "unnestexpr": 3, "unoptim": 2, "unsaf": 16, "unspil": [1, 5], "until": 19, "unwrap": 39, "up": [5, 11, 14, 19, 22, 29], "updat": [1, 2, 5, 14, 27], "upon": [4, 16, 27], "upper": 4, "uppercas": 4, "upstream": 16, "urbango": 25, "url": 1, "us": [1, 2, 3, 4, 5, 6, 7, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 35, 37, 38, 39], "usag": [1, 2, 3, 4, 5, 14, 20, 29], "user": [0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 16, 17, 19, 22, 23, 24, 31, 32], "uses_window_fram": [14, 27], "utc": 4, "utf8": [3, 4, 5], "utf8view": [3, 5], "util": [16, 29], "uuid": 4, "uv": 17, "v": 17, "v4": 4, "valid": [1, 3, 5, 6, 7, 8, 14], "valu": [1, 2, 3, 4, 5, 11, 14, 20, 22, 23, 25, 26, 27, 29], "value1": 4, "value2": 4, "value_i": 4, "value_x": 4, "valueerror": 2, "values_a": 27, "values_b": 27, "var": 4, "var_pop": [4, 20], "var_samp": [4, 20], "var_sampl": 4, "variabl": [1, 4, 5, 17, 22], "varianc": 4, "variant": [3, 5, 11, 25], "variant_nam": [3, 5], "varieti": [22, 27, 31], "variou": 32, "vastli": 17, "vec": 16, "vendorid": 26, "venomoth": 29, "venonat": 29, "venu": 23, "venufleur": 23, "venufleurmega": 23, "venusaur": [18, 23, 29, 31, 32], "venusaurmega": [18, 23, 29, 31, 32], "venv": 17, "verbos": 2, "veri": [34, 35], "verifi": 32, "verion": 31, "version": [16, 17, 31], "via": [1, 13, 16, 17, 18, 22, 25, 31, 32, 39, 40], "view": [1, 21, 23, 24, 32], "view1": 28, "viewtabl": 2, "vink": 29, "virtual": [1, 5, 17], "visual": [5, 11], "volatil": [5, 14, 27], "voltorb": [20, 29], "vulpix": 20, "wa": [1, 5], "wai": [16, 17, 20, 31, 32], "wait": 16, "want": [16, 17, 20, 23, 27, 29, 39], "wartortl": [18, 31, 32], "water": [18, 20, 23, 31, 32], "we": [2, 16, 17, 19, 20, 22, 23, 25, 26, 27, 29, 30, 32, 33], "weedl": [18, 29, 31, 32], "weight": [4, 22], "welcom": [17, 32], "well": [1, 5, 16, 17], "were": [16, 22], "what": [16, 19], "whatev": 14, "when": [1, 3, 4, 5, 9, 14, 16, 17, 18, 20, 22, 25, 27, 29, 30, 37], "when_expr": 3, "whenev": 17, "where": [3, 4, 5, 11, 20, 22, 27, 28, 29], "wherea": 22, "wherev": 16, "whether": [1, 3, 4, 5, 9, 14, 22], "which": [1, 2, 3, 4, 5, 14, 16, 17, 18, 20, 22, 26, 27, 29, 31, 33], "while": [14, 25], "who": 16, "whole": 20, "wide": [22, 30, 31], "window": [1, 3, 4, 5, 14, 20, 24], "window_fram": [3, 4, 5, 29], "windowevalu": [5, 14, 27], "windowexpr": 3, "windowfram": [3, 4, 5, 29], "windowframebound": [3, 5], "windowudf": [1, 5, 14], "wish": 16, "with_allow_ddl": [1, 5], "with_allow_dml": [1, 5], "with_allow_stat": [1, 5], "with_batch_s": [1, 5], "with_column": 2, "with_column_renam": 2, "with_create_default_catalog_and_schema": [1, 5, 30], "with_default_catalog_and_schema": [1, 5, 30], "with_disk_manager_dis": [1, 5], "with_disk_manager_o": [1, 5, 30], "with_disk_manager_specifi": [1, 5], "with_fair_spill_pool": [1, 5, 30], "with_greedy_memory_pool": [1, 5], "with_head": 2, "with_information_schema": [1, 5, 30], "with_parquet_prun": [1, 5, 30], "with_repartition_aggreg": [1, 5, 30], "with_repartition_file_min_s": [1, 5], "with_repartition_file_scan": [1, 5], "with_repartition_join": [1, 5, 30], "with_repartition_sort": [1, 5], "with_repartition_window": [1, 5, 30], "with_target_partit": [1, 5, 30], "with_temp_file_path": [1, 5], "with_unbounded_memory_pool": [1, 5], "within": [4, 7, 14, 20, 22], "within_limit": 2, "without": [4, 16, 17, 20, 23, 25, 26, 27], "won": 26, "word": 4, "work": [1, 4, 5, 23, 26, 32], "workflow": 17, "would": [3, 4, 6, 14, 16], "wrap": [1, 2, 16], "wrapper": [1, 5, 12, 17], "write": [2, 6, 13, 16, 27], "write_csv": 2, "write_json": 2, "write_parquet": 2, "written": [16, 18], "x": [4, 5, 14, 18, 23, 29, 31, 32, 40], "xor": 4, "y": [4, 14, 18, 23, 29, 31, 32], "year": 4, "years_in_posit": 22, "yellow": 31, "yellow_tripdata_2021": [19, 26], "yet": 2, "yield": 4, "you": [1, 4, 5, 14, 16, 17, 18, 19, 20, 21, 22, 23, 26, 27, 28, 29, 30, 31, 32, 33, 39], "your": [5, 14, 16, 17, 20, 21, 26, 27, 39], "yourself": 16, "z": [1, 5, 14], "zero": [2, 4, 18, 33], "zstandard": 2, "zstd": 2, "zubat": 20, "\u03c0": 4}, "titles": ["datafusion.catalog", "datafusion.context", "datafusion.dataframe", "datafusion.expr", "datafusion.functions", "datafusion", "datafusion.input.base", "datafusion.input", "datafusion.input.location", "datafusion.io", "datafusion.object_store", "datafusion.plan", "datafusion.record_batch", "datafusion.substrait", "datafusion.udf", "API Reference", "Python Extensions", "Introduction", "DataFusion in Python", "Concepts", "Aggregation", "Basic Operations", "Expressions", "Functions", "Common Operations", "Joins", "Column Selections", "User-Defined Functions", "Registering Views", "Window Functions", "Configuration", "Data Sources", "Introduction", "Arrow", "Avro", "CSV", "IO", "JSON", "Parquet", "Custom Table Provider", "SQL"], "titleterms": {"The": 16, "aggreg": [20, 27, 29], "altern": 16, "anti": 25, "api": 15, "approach": 16, "arrai": 22, "arrow": [16, 33], "attribut": [3, 5, 10, 14], "avail": 29, "avro": 34, "base": 6, "basic": 21, "boolean": 22, "build": 17, "cast": 23, "catalog": 0, "class": [0, 1, 2, 3, 5, 6, 7, 8, 11, 12, 13, 14], "code": 17, "column": [22, 26], "commit": 17, "common": 24, "concept": 19, "condit": 23, "configur": 30, "content": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "context": [1, 19], "creat": 31, "csv": 35, "custom": [31, 39], "data": 31, "datafram": [2, 19, 31], "datafus": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 33], "defin": 27, "delta": 31, "depend": 17, "detail": 16, "develop": 17, "distinct": 20, "exampl": 18, "export": 33, "expr": 3, "express": [19, 22], "extens": 16, "ffi": 16, "file": 31, "filter": 20, "frame": 29, "from": [16, 33], "full": 25, "function": [4, 5, 9, 20, 22, 23, 27, 29], "guidelin": 17, "hook": 17, "how": 17, "iceberg": 31, "implement": 16, "import": 33, "improv": 17, "inner": 25, "input": [6, 7, 8], "inspir": 16, "instal": [17, 18, 32], "introduct": [17, 32], "io": [9, 36], "issu": 16, "join": 25, "json": 37, "lake": 31, "left": 25, "librari": 31, "liter": 22, "local": 31, "locat": 8, "mathemat": 23, "memori": 31, "modul": [0, 1, 2, 3, 4, 6, 8, 9, 10, 11, 12, 13, 14], "null": [20, 29], "object": 31, "object_stor": 10, "oper": [21, 24], "option": 27, "order": [20, 29], "other": [23, 31], "packag": [5, 7], "paramet": [20, 29], "parquet": 38, "partit": 29, "plan": 11, "pre": 17, "primari": 16, "provid": [31, 39], "python": [16, 17, 18], "record_batch": 12, "refer": 15, "regist": 28, "return": 4, "run": 17, "rust": 17, "scalar": 27, "select": 26, "semi": 25, "separ": 17, "session": 19, "set": [20, 29], "sourc": 31, "speed": 17, "sql": 40, "statu": 16, "store": 31, "string": 23, "struct": 22, "submodul": [5, 7], "substrait": 13, "tabl": [31, 39], "tempor": 23, "treatment": [20, 29], "udf": 14, "udwf": 27, "updat": 17, "user": 27, "view": 28, "window": [27, 29], "work": 16}}) \ No newline at end of file
diff --git a/user-guide/basics.html b/user-guide/basics.html index b7d6122..ab96577 100644 --- a/user-guide/basics.html +++ b/user-guide/basics.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/common-operations/aggregations.html b/user-guide/common-operations/aggregations.html index 941c82a..9235368 100644 --- a/user-guide/common-operations/aggregations.html +++ b/user-guide/common-operations/aggregations.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -442,17 +457,18 @@ <span class="o">+----------+-------+--------------+-----------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Count</span> <span class="o">|</span> <span class="n">Median</span> <span class="n">Speed</span> <span class="o">|</span> <span class="mi">90</span><span class="o">%</span> <span class="n">Speed</span> <span class="o">|</span> <span class="o">+----------+-------+--------------+-----------+</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> <span class="mi">71</span> <span class="o">|</span> <span class="mi">110</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mi">11</span> <span class="o">|</span> <span class="mi">63</span> <span class="o">|</span> <span class="mi">107</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="mi">12</span> <span class="o">|</span> <span class="mi">55</span> <span class="o">|</span> <span class="mi">85</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="mi">136</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">47</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> <span class="mi">71</span> <span class="o">|</span> <span class="mi">110</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="mi">55</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="mi">91</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="mi">21</span> <span class="o">|</span> <span class="mi">70</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="mi">40</span> <span class="o">|</span> <span class="mi">112</span> <span class="o">|</span> <span class="o">+----------+-------+--------------+-----------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>When the <code class="code docutils literal notranslate"><span class="pre">group_by</span></code> list is empty the aggregation is done over the whole <a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>. @@ -464,20 +480,21 @@ <span class="o">...</span><span class="p">:</span> <span class="n">Out</span><span class="p">[</span><span class="mi">9</span><span class="p">]:</span> <span class="n">DataFrame</span><span class="p">()</span> -<span class="o">+----------+-----------+-------------------+-----------+</span> -<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Max</span> <span class="n">Speed</span> <span class="o">|</span> <span class="n">Avg</span> <span class="n">Speed</span> <span class="o">|</span> <span class="n">Min</span> <span class="n">Speed</span> <span class="o">|</span> -<span class="o">+----------+-----------+-------------------+-----------+</span> -<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="mf">54.23076923076923</span> <span class="o">|</span> <span class="mi">30</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="mi">105</span> <span class="o">|</span> <span class="mf">86.28571428571429</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="mi">115</span> <span class="o">|</span> <span class="mf">67.25806451612904</span> <span class="o">|</span> <span class="mi">15</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="mi">120</span> <span class="o">|</span> <span class="mf">58.125</span> <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fighting</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="mf">66.14285714285714</span> <span class="o">|</span> <span class="mi">35</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="mi">150</span> <span class="o">|</span> <span class="mf">99.25</span> <span class="o">|</span> <span class="mi">42</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Rock</span> <span class="o">|</span> <span class="mi">150</span> <span class="o">|</span> <span class="mf">67.5</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ghost</span> <span class="o">|</span> <span class="mi">130</span> <span class="o">|</span> <span class="mf">103.75</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Dragon</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="mf">66.66666666666667</span> <span class="o">|</span> <span class="mi">50</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mi">121</span> <span class="o">|</span> <span class="mf">72.75</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> -<span class="o">+----------+-----------+-------------------+-----------+</span> +<span class="o">+----------+-----------+--------------------+-----------+</span> +<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Max</span> <span class="n">Speed</span> <span class="o">|</span> <span class="n">Avg</span> <span class="n">Speed</span> <span class="o">|</span> <span class="n">Min</span> <span class="n">Speed</span> <span class="o">|</span> +<span class="o">+----------+-----------+--------------------+-----------+</span> +<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mi">145</span> <span class="o">|</span> <span class="mf">66.78571428571429</span> <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="mf">58.785714285714285</span> <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="mi">140</span> <span class="o">|</span> <span class="mf">98.88888888888889</span> <span class="o">|</span> <span class="mi">45</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> <span class="mf">47.5</span> <span class="o">|</span> <span class="mi">35</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mi">121</span> <span class="o">|</span> <span class="mf">72.75</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="mf">90.0</span> <span class="o">|</span> <span class="mi">85</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="mf">54.23076923076923</span> <span class="o">|</span> <span class="mi">30</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="mi">105</span> <span class="o">|</span> <span class="mf">86.28571428571429</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="mi">115</span> <span class="o">|</span> <span class="mf">67.25806451612904</span> <span class="o">|</span> <span class="mi">15</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="mi">120</span> <span class="o">|</span> <span class="mf">58.125</span> <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> +<span class="o">+----------+-----------+--------------------+-----------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>More than one column can be used for grouping</p> @@ -502,6 +519,7 @@ <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="mf">96.66666666666667</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="n">Dragon</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="mf">100.0</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="o">+----------+---------+-----------+--------------------+-----------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <section id="setting-parameters"> @@ -534,17 +552,18 @@ <span class="o">+----------+-----------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Smallest</span> <span class="n">Attack</span> <span class="o">|</span> <span class="o">+----------+-----------------+</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="n">Chansey</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Jynx</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="n">Metapod</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="n">Zubat</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="n">Voltorb</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="n">Clefairy</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="n">Chansey</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Jynx</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="n">Exeggcute</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="n">Vulpix</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="n">Magikarp</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="n">Cubone</span> <span class="o">|</span> <span class="o">+----------+-----------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -559,17 +578,18 @@ <span class="o">+----------+--------------------------------------------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">2</span> <span class="n">List</span> <span class="o">|</span> <span class="o">+----------+--------------------------------------------------+</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Poison</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Grass</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Grass</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Ground</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Steel</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Steel</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="p">[]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Psychic</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Dragon</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Ice</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span class="p">,</span> <span class="n">Psychic</span><span class="p">,</span> <span class="n">Fighting</span><span class="p">,</span> <span class="n">Dark</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Fairy</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span class="p">,</span> <span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="p">[</span><span class="n">Dragon</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Fighting</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span class="p">,</span> <span class="n">Dark</span><span class="p">,</span> <span class="n">Ice</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="p">[</span><span class="n">Rock</span><span class="p">,</span> <span class="p">]</span> <span class="o">|</span> <span class="o">+----------+--------------------------------------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>In the output of the above we can see that there are some <code class="docutils literal notranslate"><span class="pre">Type</span> <span class="pre">1</span></code> for which the <code class="docutils literal notranslate"><span class="pre">Type</span> <span class="pre">2</span></code> entry @@ -582,17 +602,18 @@ <span class="o">+----------+------------------------------------------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">2</span> <span class="n">List</span> <span class="o">|</span> <span class="o">+----------+------------------------------------------------+</span> -<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[</span><span class="n">Steel</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Ground</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Steel</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Fairy</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Dragon</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span class="p">,</span> <span class="n">Fighting</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Ice</span><span class="p">,</span> <span class="n">Dark</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Fighting</span><span class="p">,</span> <span class="n">Dark</span><span class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span class="n">Ice</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Rock</span> <span class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span class="p">,</span> <span class="n">Water</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ghost</span> <span class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> <span class="o">+----------+------------------------------------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> <span class="n">In</span> <span class="p">[</span><span class="mi">16</span><span class="p">]:</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">([</span><span class="n">col_type_1</span><span class="p">],</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">array_agg</span><span class="p">(</span><span class="n">col_type_2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="n">col_type_2</span><span class="o">.</span><span class="n">is_not_null</span><span class="p">())</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"Type 2 List"</span><span class="p">)])</span> <span class="n">Out</span><span class="p">[</span><span class="mi">16</span><span class="p">]:</span> @@ -600,17 +621,18 @@ <span class="o">+----------+------------------------------------------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">2</span> <span class="n">List</span> <span class="o">|</span> <span class="o">+----------+------------------------------------------------+</span> +<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Ground</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[</span><span class="n">Steel</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Ground</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Steel</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span class="p">,</span> <span class="n">Poison</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="p">[</span><span class="n">Dragon</span><span class="p">,</span> <span class="n">Flying</span><span class="p">]</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Ice</span><span class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Fighting</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">,</span> <span class="n">Dark</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span class="p">,</span> <span class="n">Dragon</span><span class="p">]</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="p">[</span><span class="n">Ice</span><span class="p">,</span> <span class="n">Dark</span><span class="p">,</span> <span class="n">Fighting</span><span class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="p">[</span><span class="n">Rock</span><span class="p">]</span> <span class="o">|</span> <span class="o">+----------+------------------------------------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>Which approach you take should depend on your use case.</p> @@ -635,17 +657,18 @@ <span class="o">+----------+----------------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Lowest</span> <span class="n">Attack</span> <span class="n">Type</span> <span class="mi">2</span> <span class="o">|</span> <span class="o">+----------+----------------------+</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="o">|</span> <span class="o">+----------+----------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> <span class="n">In</span> <span class="p">[</span><span class="mi">19</span><span class="p">]:</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">([</span><span class="n">col_type_1</span><span class="p">],</span> <span class="p">[</span> <span class="o">....</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">first_value</span><span class="p">(</span> @@ -659,17 +682,18 @@ <span class="o">+----------+----------------------+</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Lowest</span> <span class="n">Attack</span> <span class="n">Type</span> <span class="mi">2</span> <span class="o">|</span> <span class="o">+----------+----------------------+</span> -<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="n">Steel</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="n">Flying</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="n">Rock</span> <span class="o">|</span> <span class="o">+----------+----------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -693,13 +717,14 @@ <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="mf">58.785714285714285</span> <span class="o">|</span> <span class="mf">48.0</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Electric</span> <span class="o">|</span> <span class="mf">98.88888888888889</span> <span class="o">|</span> <span class="mf">72.5</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fairy</span> <span class="o">|</span> <span class="mf">47.5</span> <span class="o">|</span> <span class="mf">35.0</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">72.75</span> <span class="o">|</span> <span class="mf">52.8</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mf">90.0</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Grass</span> <span class="o">|</span> <span class="mf">54.23076923076923</span> <span class="o">|</span> <span class="mf">42.5</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Fire</span> <span class="o">|</span> <span class="mf">86.28571428571429</span> <span class="o">|</span> <span class="mf">65.0</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Water</span> <span class="o">|</span> <span class="mf">67.25806451612904</span> <span class="o">|</span> <span class="mf">63.833333333333336</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Ground</span> <span class="o">|</span> <span class="mf">58.125</span> <span class="o">|</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fighting</span> <span class="o">|</span> <span class="mf">66.14285714285714</span> <span class="o">|</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Psychic</span> <span class="o">|</span> <span class="mf">99.25</span> <span class="o">|</span> <span class="mf">81.75</span> <span class="o">|</span> <span class="o">+----------+--------------------+----------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section>
diff --git a/user-guide/common-operations/basic-info.html b/user-guide/common-operations/basic-info.html index e587ee1..f7c6612 100644 --- a/user-guide/common-operations/basic-info.html +++ b/user-guide/common-operations/basic-info.html
@@ -35,7 +35,7 @@ <link rel="index" title="Index" href="../../genindex.html" /> <link rel="search" title="Search" href="../../search.html" /> <link rel="next" title="Column Selections" href="select-and-filter.html" /> - <link rel="prev" title="Common Operations" href="index.html" /> + <link rel="prev" title="Registering Views" href="views.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -127,6 +127,11 @@ </i> </label> <ul class="current"> + <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> <li class="toctree-l2 current active"> <a class="current reference internal" href="#"> Basic Operations @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -399,11 +414,11 @@ <span class="o">+-----+---------+--------+--------+</span> <span class="o">|</span> <span class="n">nrs</span> <span class="o">|</span> <span class="n">names</span> <span class="o">|</span> <span class="n">random</span> <span class="o">|</span> <span class="n">groups</span> <span class="o">|</span> <span class="o">+-----+---------+--------+--------+</span> -<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">python</span> <span class="o">|</span> <span class="mi">325</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mi">43</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="n">java</span> <span class="o">|</span> <span class="mi">169</span> <span class="o">|</span> <span class="n">B</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="n">haskell</span> <span class="o">|</span> <span class="mi">11</span> <span class="o">|</span> <span class="n">C</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span class="mi">362</span> <span class="o">|</span> <span class="n">B</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">python</span> <span class="o">|</span> <span class="mi">832</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mi">669</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="n">java</span> <span class="o">|</span> <span class="mi">803</span> <span class="o">|</span> <span class="n">B</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="n">haskell</span> <span class="o">|</span> <span class="mi">531</span> <span class="o">|</span> <span class="n">C</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span class="mi">774</span> <span class="o">|</span> <span class="n">B</span> <span class="o">|</span> <span class="o">+-----+---------+--------+--------+</span> </pre></div> </div> @@ -414,8 +429,8 @@ <span class="o">+-----+--------+--------+--------+</span> <span class="o">|</span> <span class="n">nrs</span> <span class="o">|</span> <span class="n">names</span> <span class="o">|</span> <span class="n">random</span> <span class="o">|</span> <span class="n">groups</span> <span class="o">|</span> <span class="o">+-----+--------+--------+--------+</span> -<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">python</span> <span class="o">|</span> <span class="mi">325</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mi">43</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">python</span> <span class="o">|</span> <span class="mi">832</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mi">669</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> <span class="o">+-----+--------+--------+--------+</span> </pre></div> </div> @@ -433,11 +448,11 @@ <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="n">In</span> <span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="n">df</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> <span class="n">Out</span><span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="n">nrs</span> <span class="n">names</span> <span class="n">random</span> <span class="n">groups</span> -<span class="mi">0</span> <span class="mi">1</span> <span class="n">python</span> <span class="mi">325</span> <span class="n">A</span> -<span class="mi">1</span> <span class="mi">2</span> <span class="n">ruby</span> <span class="mi">43</span> <span class="n">A</span> -<span class="mi">2</span> <span class="mi">3</span> <span class="n">java</span> <span class="mi">169</span> <span class="n">B</span> -<span class="mi">3</span> <span class="mi">4</span> <span class="n">haskell</span> <span class="mi">11</span> <span class="n">C</span> -<span class="mi">4</span> <span class="mi">5</span> <span class="n">go</span> <span class="mi">362</span> <span class="n">B</span> +<span class="mi">0</span> <span class="mi">1</span> <span class="n">python</span> <span class="mi">832</span> <span class="n">A</span> +<span class="mi">1</span> <span class="mi">2</span> <span class="n">ruby</span> <span class="mi">669</span> <span class="n">A</span> +<span class="mi">2</span> <span class="mi">3</span> <span class="n">java</span> <span class="mi">803</span> <span class="n">B</span> +<span class="mi">3</span> <span class="mi">4</span> <span class="n">haskell</span> <span class="mi">531</span> <span class="n">C</span> +<span class="mi">4</span> <span class="mi">5</span> <span class="n">go</span> <span class="mi">774</span> <span class="n">B</span> </pre></div> </div> <p><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.describe" title="datafusion.dataframe.DataFrame.describe"><code class="xref py py-func docutils literal notranslate"><span class="pre">describe()</span></code></a> shows a quick statistic summary of your data:</p> @@ -449,11 +464,11 @@ <span class="o">+------------+--------------------+-------+--------------------+--------+</span> <span class="o">|</span> <span class="n">count</span> <span class="o">|</span> <span class="mf">5.0</span> <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="mf">5.0</span> <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="o">|</span> <span class="n">null_count</span> <span class="o">|</span> <span class="mf">0.0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mf">0.0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">mean</span> <span class="o">|</span> <span class="mf">3.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">182.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">std</span> <span class="o">|</span> <span class="mf">1.5811388300841898</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">159.35808733791956</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> -<span class="o">|</span> <span class="nb">min</span> <span class="o">|</span> <span class="mf">1.0</span> <span class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span class="mf">11.0</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> -<span class="o">|</span> <span class="nb">max</span> <span class="o">|</span> <span class="mf">5.0</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mf">362.0</span> <span class="o">|</span> <span class="n">C</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">median</span> <span class="o">|</span> <span class="mf">3.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">169.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">mean</span> <span class="o">|</span> <span class="mf">3.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">721.8</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">std</span> <span class="o">|</span> <span class="mf">1.5811388300841898</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">123.14097612086726</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> +<span class="o">|</span> <span class="nb">min</span> <span class="o">|</span> <span class="mf">1.0</span> <span class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span class="mf">531.0</span> <span class="o">|</span> <span class="n">A</span> <span class="o">|</span> +<span class="o">|</span> <span class="nb">max</span> <span class="o">|</span> <span class="mf">5.0</span> <span class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span class="mf">832.0</span> <span class="o">|</span> <span class="n">C</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">median</span> <span class="o">|</span> <span class="mf">3.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="mf">774.0</span> <span class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span class="o">+------------+--------------------+-------+--------------------+--------+</span> </pre></div> </div> @@ -465,11 +480,11 @@ <!-- Previous / next buttons --> <div class='prev-next-area'> - <a class='left-prev' id="prev-link" href="index.html" title="previous page"> + <a class='left-prev' id="prev-link" href="views.html" title="previous page"> <i class="fas fa-angle-left"></i> <div class="prev-next-info"> <p class="prev-next-subtitle">previous</p> - <p class="prev-next-title">Common Operations</p> + <p class="prev-next-title">Registering Views</p> </div> </a> <a class='right-next' id="next-link" href="select-and-filter.html" title="next page">
diff --git a/user-guide/common-operations/expressions.html b/user-guide/common-operations/expressions.html index 6542c9e..0fe3e29 100644 --- a/user-guide/common-operations/expressions.html +++ b/user-guide/common-operations/expressions.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/common-operations/functions.html b/user-guide/common-operations/functions.html index 0a8ba7e..803c266 100644 --- a/user-guide/common-operations/functions.html +++ b/user-guide/common-operations/functions.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -490,17 +505,18 @@ <span class="o">+--------------------------------+</span> <span class="o">|</span> <span class="n">now</span><span class="p">()</span> <span class="o">|</span> <span class="o">+--------------------------------+</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> -<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">02</span><span class="o">-</span><span class="mi">24</span><span class="n">T20</span><span class="p">:</span><span class="mi">30</span><span class="p">:</span><span class="mf">22.735522831</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> +<span class="o">|</span> <span class="mi">2025</span><span class="o">-</span><span class="mi">03</span><span class="o">-</span><span class="mi">30</span><span class="n">T17</span><span class="p">:</span><span class="mi">49</span><span class="p">:</span><span class="mf">00.536576280</span><span class="n">Z</span> <span class="o">|</span> <span class="o">+--------------------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>Convert to timestamps using <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#datafusion.functions.to_timestamp" title="datafusion.functions.to_timestamp"><code class="xref py py-func docutils literal notranslate"><span class="pre">to_timestamp()</span></code></a></p> @@ -521,6 +537,7 @@ <span class="o">|</span> <span class="mi">1970</span><span class="o">-</span><span class="mi">01</span><span class="o">-</span><span class="mi">01</span><span class="n">T00</span><span class="p">:</span><span class="mi">10</span><span class="p">:</span><span class="mi">34</span> <span class="o">|</span> <span class="o">|</span> <span class="mi">1970</span><span class="o">-</span><span class="mi">01</span><span class="o">-</span><span class="mi">01</span><span class="n">T00</span><span class="p">:</span><span class="mi">05</span><span class="p">:</span><span class="mi">14</span> <span class="o">|</span> <span class="o">+---------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>Extracting parts of a date using <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#datafusion.functions.date_part" title="datafusion.functions.date_part"><code class="xref py py-func docutils literal notranslate"><span class="pre">date_part()</span></code></a> (alias <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#datafusion.functions.extract" title="datafusion.functions.extract"><code class="xref py py-func docutils literal notranslate"><span class="pre">extract()</span></code></a>)</p> @@ -545,6 +562,7 @@ <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">+-------+-----+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -574,6 +592,7 @@ <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> <span class="n">charizardmega</span> <span class="n">charizard</span> <span class="n">y</span> <span class="o">|</span> <span class="n">Char</span> <span class="o">|</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">squirtle</span> <span class="o">|</span> <span class="n">Squi</span> <span class="o">|</span> <span class="o">+-----+---------------------------+------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>This also includes the functions for regular expressions like <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#datafusion.functions.regexp_replace" title="datafusion.functions.regexp_replace"><code class="xref py py-func docutils literal notranslate"><span class="pre">regexp_replace()</span></code></a> and <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#datafusion.functions.regexp_match" title="datafusion.functions.regexp_match"><code class="xref py py-func docutils literal notranslate"><span class="pre">regexp_match()</span></code></a></p> @@ -598,6 +617,7 @@ <span class="o">|</span> <span class="p">[</span><span class="n">Char</span><span class="p">]</span> <span class="o">|</span> <span class="n">CharizardMega</span> <span class="n">Charizard</span> <span class="n">Y</span> <span class="o">|</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Squirtle</span> <span class="o">|</span> <span class="o">+---------+---------------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -625,6 +645,7 @@ <span class="o">|</span> <span class="mf">634.0</span> <span class="o">|</span> <span class="mi">634</span> <span class="o">|</span> <span class="o">|</span> <span class="mf">314.0</span> <span class="o">|</span> <span class="mi">314</span> <span class="o">|</span> <span class="o">+----------------+--------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section>
diff --git a/user-guide/common-operations/index.html b/user-guide/common-operations/index.html index e66669a..a8f5abc 100644 --- a/user-guide/common-operations/index.html +++ b/user-guide/common-operations/index.html
@@ -34,7 +34,7 @@ <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> <link rel="index" title="Index" href="../../genindex.html" /> <link rel="search" title="Search" href="../../search.html" /> - <link rel="next" title="Basic Operations" href="basic-info.html" /> + <link rel="next" title="Registering Views" href="views.html" /> <link rel="prev" title="Data Sources" href="../data-sources.html" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="docsearch:language" content="en"> @@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -381,6 +396,7 @@ <p>The contents of this section are designed to guide a new user through how to use DataFusion.</p> <div class="toctree-wrapper compound"> <ul> +<li class="toctree-l1"><a class="reference internal" href="views.html">Registering Views</a></li> <li class="toctree-l1"><a class="reference internal" href="basic-info.html">Basic Operations</a></li> <li class="toctree-l1"><a class="reference internal" href="select-and-filter.html">Column Selections</a></li> <li class="toctree-l1"><a class="reference internal" href="expressions.html">Expressions</a><ul> @@ -443,10 +459,10 @@ <p class="prev-next-title">Data Sources</p> </div> </a> - <a class='right-next' id="next-link" href="basic-info.html" title="next page"> + <a class='right-next' id="next-link" href="views.html" title="next page"> <div class="prev-next-info"> <p class="prev-next-subtitle">next</p> - <p class="prev-next-title">Basic Operations</p> + <p class="prev-next-title">Registering Views</p> </div> <i class="fas fa-angle-right"></i> </a>
diff --git a/user-guide/common-operations/joins.html b/user-guide/common-operations/joins.html index 026f289..880447f 100644 --- a/user-guide/common-operations/joins.html +++ b/user-guide/common-operations/joins.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/common-operations/select-and-filter.html b/user-guide/common-operations/select-and-filter.html index 63ebfb1..1a01824 100644 --- a/user-guide/common-operations/select-and-filter.html +++ b/user-guide/common-operations/select-and-filter.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -405,6 +420,7 @@ <span class="o">|</span> <span class="mf">9.1</span> <span class="o">|</span> <span class="mf">1.0</span> <span class="o">|</span> <span class="o">|</span> <span class="mf">2.7</span> <span class="o">|</span> <span class="mf">2.0</span> <span class="o">|</span> <span class="o">+---------------+-----------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>For mathematical or logical operations use <a class="reference internal" href="../../autoapi/datafusion/index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">col()</span></code></a> to select columns, and give meaningful names to the resulting @@ -428,6 +444,7 @@ <span class="o">|</span> <span class="mf">0.0</span> <span class="o">|</span> <span class="o">|</span> <span class="mf">3.15</span> <span class="o">|</span> <span class="o">+-----------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <div class="admonition warning"> @@ -454,6 +471,7 @@ <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">+----------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <p>To combine it with literal values use the <a class="reference internal" href="../../autoapi/datafusion/index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a></p> @@ -478,6 +496,7 @@ <span class="o">|</span> <span class="n">true</span> <span class="o">|</span> <span class="o">|</span> <span class="n">false</span> <span class="o">|</span> <span class="o">+--------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section>
diff --git a/user-guide/common-operations/udf-and-udfa.html b/user-guide/common-operations/udf-and-udfa.html index 38b15b1..92d9457 100644 --- a/user-guide/common-operations/udf-and-udfa.html +++ b/user-guide/common-operations/udf-and-udfa.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/common-operations/views.html b/user-guide/common-operations/views.html new file mode 100644 index 0000000..0ffc23d --- /dev/null +++ b/user-guide/common-operations/views.html
@@ -0,0 +1,487 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> + + <title>Registering Views — Apache Arrow DataFusion documentation</title> + + <link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> +<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> + + + <link rel="stylesheet" + href="../../_static/vendor/fontawesome/5.13.0/css/all.min.css"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2"> + <link rel="preload" as="font" type="font/woff2" crossorigin + href="../../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2"> + + + + + + <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" /> + <link rel="stylesheet" type="text/css" href="../../_static/styles/pydata-sphinx-theme.css?v=1140d252" /> + <link rel="stylesheet" type="text/css" href="../../_static/graphviz.css?v=4ae1632d" /> + <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=dca7052a" /> + + <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"> + + <script src="../../_static/documentation_options.js?v=8a448e45"></script> + <script src="../../_static/doctools.js?v=9bcbadda"></script> + <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> + <link rel="index" title="Index" href="../../genindex.html" /> + <link rel="search" title="Search" href="../../search.html" /> + <link rel="next" title="Basic Operations" href="basic-info.html" /> + <link rel="prev" title="Common Operations" href="index.html" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <meta name="docsearch:language" content="en"> + + + <!-- Google Analytics --> + + </head> + <body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80"> + + <div class="container-fluid" id="banner"></div> + + + + + <div class="container-xl"> + <div class="row"> + + + <!-- Only show if we have sidebars configured, else just a small margin --> + <div class="col-12 col-md-3 bd-sidebar"> + <div class="sidebar-start-items"> +<a class="navbar-brand" href="../../index.html"> + <img src="../../_static/images/2x_bgwhite_original.png" class="logo" alt="logo"> +</a> + +<form class="bd-search d-flex align-items-center" action="../../search.html" method="get"> + <i class="icon fas fa-search"></i> + <input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" > +</form> + +<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation"> + <div class="bd-toc-item active"> + + <p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + LINKS + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python"> + Github and Issue Tracker + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/"> + Rust's API Docs + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md"> + Code of conduct + </a> + </li> + <li class="toctree-l1"> + <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples"> + Examples + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + USER GUIDE + </span> +</p> +<ul class="current nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="../introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../basics.html"> + Concepts + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../data-sources.html"> + Data Sources + </a> + </li> + <li class="toctree-l1 current active has-children"> + <a class="reference internal" href="index.html"> + Common Operations + </a> + <input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/> + <label for="toctree-checkbox-1"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul class="current"> + <li class="toctree-l2 current active"> + <a class="current reference internal" href="#"> + Registering Views + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="basic-info.html"> + Basic Operations + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="select-and-filter.html"> + Column Selections + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="expressions.html"> + Expressions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="joins.html"> + Joins + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="functions.html"> + Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="aggregations.html"> + Aggregation + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="windows.html"> + Window Functions + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="udf-and-udfa.html"> + User-Defined Functions + </a> + </li> + </ul> + </li> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../io/index.html"> + IO + </a> + <input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/> + <label for="toctree-checkbox-2"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2"> + <a class="reference internal" href="../io/arrow.html"> + Arrow + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../io/avro.html"> + Avro + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../io/csv.html"> + CSV + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../io/json.html"> + JSON + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../io/parquet.html"> + Parquet + </a> + </li> + <li class="toctree-l2"> + <a class="reference internal" href="../io/table_provider.html"> + Custom Table Provider + </a> + </li> + </ul> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../configuration.html"> + Configuration + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../sql.html"> + SQL + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + CONTRIBUTOR GUIDE + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/introduction.html"> + Introduction + </a> + </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> +</ul> +<p aria-level="2" class="caption" role="heading"> + <span class="caption-text"> + API + </span> +</p> +<ul class="nav bd-sidenav"> + <li class="toctree-l1 has-children"> + <a class="reference internal" href="../../autoapi/index.html"> + API Reference + </a> + <input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/> + <label for="toctree-checkbox-3"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l2 has-children"> + <a class="reference internal" href="../../autoapi/datafusion/index.html"> + datafusion + </a> + <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/> + <label for="toctree-checkbox-4"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/catalog/index.html"> + datafusion.catalog + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/context/index.html"> + datafusion.context + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html"> + datafusion.dataframe + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/expr/index.html"> + datafusion.expr + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/functions/index.html"> + datafusion.functions + </a> + </li> + <li class="toctree-l3 has-children"> + <a class="reference internal" href="../../autoapi/datafusion/input/index.html"> + datafusion.input + </a> + <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/> + <label for="toctree-checkbox-5"> + <i class="fas fa-chevron-down"> + </i> + </label> + <ul> + <li class="toctree-l4"> + <a class="reference internal" href="../../autoapi/datafusion/input/base/index.html"> + datafusion.input.base + </a> + </li> + <li class="toctree-l4"> + <a class="reference internal" href="../../autoapi/datafusion/input/location/index.html"> + datafusion.input.location + </a> + </li> + </ul> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> + datafusion.object_store + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/plan/index.html"> + datafusion.plan + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/record_batch/index.html"> + datafusion.record_batch + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/substrait/index.html"> + datafusion.substrait + </a> + </li> + <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/udf/index.html"> + datafusion.udf + </a> + </li> + </ul> + </li> + </ul> + </li> +</ul> + + + </div> +</nav> + </div> + <div class="sidebar-end-items"> + </div> + </div> + + + + + <div class="d-none d-xl-block col-xl-2 bd-toc"> + + + <div class="toc-item"> + + +<nav id="bd-toc-nav"> + +</nav> + </div> + + <div class="toc-item"> + + </div> + + + </div> + + + + + + + <main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main"> + + <div> + + <section id="registering-views"> +<h1>Registering Views<a class="headerlink" href="#registering-views" title="Link to this heading">¶</a></h1> +<p>You can use the context’s <code class="docutils literal notranslate"><span class="pre">register_view</span></code> method to register a DataFrame as a view</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">literal</span> + +<span class="c1"># Create a DataFusion context</span> +<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> + +<span class="c1"># Create sample data</span> +<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="mi">50</span><span class="p">]}</span> + +<span class="c1"># Create a DataFrame from the dictionary</span> +<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"my_table"</span><span class="p">)</span> + +<span class="c1"># Filter the DataFrame (for example, keep rows where a > 2)</span> +<span class="n">df_filtered</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">literal</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span> + +<span class="c1"># Register the dataframe as a view with the context</span> +<span class="n">ctx</span><span class="o">.</span><span class="n">register_view</span><span class="p">(</span><span class="s2">"view1"</span><span class="p">,</span> <span class="n">df_filtered</span><span class="p">)</span> + +<span class="c1"># Now run a SQL query against the registered view</span> +<span class="n">df_view</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM view1"</span><span class="p">)</span> + +<span class="c1"># Collect the results</span> +<span class="n">results</span> <span class="o">=</span> <span class="n">df_view</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span> + +<span class="c1"># Convert results to a list of dictionaries for display</span> +<span class="n">result_dicts</span> <span class="o">=</span> <span class="p">[</span><span class="n">batch</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">results</span><span class="p">]</span> + +<span class="nb">print</span><span class="p">(</span><span class="n">result_dicts</span><span class="p">)</span> +</pre></div> +</div> +<p>This will output:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="s1">'a'</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s1">'b'</span><span class="p">:</span> <span class="p">[</span><span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="mi">50</span><span class="p">]}]</span> +</pre></div> +</div> +</section> + + + </div> + + + <!-- Previous / next buttons --> +<div class='prev-next-area'> + <a class='left-prev' id="prev-link" href="index.html" title="previous page"> + <i class="fas fa-angle-left"></i> + <div class="prev-next-info"> + <p class="prev-next-subtitle">previous</p> + <p class="prev-next-title">Common Operations</p> + </div> + </a> + <a class='right-next' id="next-link" href="basic-info.html" title="next page"> + <div class="prev-next-info"> + <p class="prev-next-subtitle">next</p> + <p class="prev-next-title">Basic Operations</p> + </div> + <i class="fas fa-angle-right"></i> + </a> +</div> + + </main> + + + </div> + </div> + + <script src="../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script> + +<!-- Based on pydata_sphinx_theme/footer.html --> +<footer class="footer mt-5 mt-md-0"> + <div class="container"> + + <div class="footer-item"> + <p class="copyright"> + © Copyright 2019-2024, Apache Software Foundation.<br> +</p> + </div> + + <div class="footer-item"> + <p class="sphinx-version"> +Created using <a href="http://sphinx-doc.org/">Sphinx</a> 8.1.3.<br> +</p> + </div> + + <div class="footer-item"> + <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> + <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> + </div> + </div> +</footer> + + + </body> +</html> \ No newline at end of file
diff --git a/user-guide/common-operations/windows.html b/user-guide/common-operations/windows.html index 0faeda2..4b2be12 100644 --- a/user-guide/common-operations/windows.html +++ b/user-guide/common-operations/windows.html
@@ -128,6 +128,11 @@ </label> <ul class="current"> <li class="toctree-l2"> + <a class="reference internal" href="views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -460,6 +475,7 @@ <span class="o">|</span> <span class="n">CharizardMega</span> <span class="n">Charizard</span> <span class="n">Y</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Squirtle</span> <span class="o">|</span> <span class="mi">43</span> <span class="o">|</span> <span class="mi">100</span> <span class="o">|</span> <span class="o">+---------------------------+-------+----------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> <section id="setting-parameters"> @@ -494,6 +510,7 @@ <span class="o">|</span> <span class="n">Beedrill</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mi">9</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Parasect</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mi">10</span> <span class="o">|</span> <span class="o">+------------+--------+--------+------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -530,6 +547,7 @@ <span class="o">|</span> <span class="n">Machop</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="n">Fighting</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Mankey</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="n">Fighting</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="o">+-----------+--------+----------+------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -585,6 +603,7 @@ <span class="o">|</span> <span class="n">Oddish</span> <span class="o">|</span> <span class="mi">30</span> <span class="o">|</span> <span class="mf">30.0</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Parasect</span> <span class="o">|</span> <span class="mi">30</span> <span class="o">|</span> <span class="mf">30.0</span> <span class="o">|</span> <span class="o">+------------+-------+--------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -634,6 +653,7 @@ <span class="o">|</span> <span class="n">Beedrill</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Pinsir</span> <span class="o">|</span> <span class="o">|</span> <span class="n">Poison</span> <span class="o">|</span> <span class="o">|</span> <span class="o">+------------+--------+--------------+----------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section> @@ -658,20 +678,21 @@ <span class="o">....</span><span class="p">:</span> <span class="n">Out</span><span class="p">[</span><span class="mi">13</span><span class="p">]:</span> <span class="n">DataFrame</span><span class="p">()</span> -<span class="o">+---------------------+--------+--------+----------------+</span> -<span class="o">|</span> <span class="n">Name</span> <span class="o">|</span> <span class="n">Attack</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Average</span> <span class="n">Attack</span> <span class="o">|</span> -<span class="o">+---------------------+--------+--------+----------------+</span> -<span class="o">|</span> <span class="n">Jynx</span> <span class="o">|</span> <span class="mi">50</span> <span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mf">67.5</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Articuno</span> <span class="o">|</span> <span class="mi">85</span> <span class="o">|</span> <span class="n">Ice</span> <span class="o">|</span> <span class="mf">67.5</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Pidgey</span> <span class="o">|</span> <span class="mi">45</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Pidgeotto</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Pidgeot</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">PidgeotMega</span> <span class="n">Pidgeot</span> <span class="o">|</span> <span class="mi">80</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Rattata</span> <span class="o">|</span> <span class="mi">56</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Raticate</span> <span class="o">|</span> <span class="mi">81</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Spearow</span> <span class="o">|</span> <span class="mi">60</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">|</span> <span class="n">Fearow</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="n">Normal</span> <span class="o">|</span> <span class="mf">70.625</span> <span class="o">|</span> -<span class="o">+---------------------+--------+--------+----------------+</span> +<span class="o">+-----------------------+--------+--------+-------------------+</span> +<span class="o">|</span> <span class="n">Name</span> <span class="o">|</span> <span class="n">Attack</span> <span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Average</span> <span class="n">Attack</span> <span class="o">|</span> +<span class="o">+-----------------------+--------+--------+-------------------+</span> +<span class="o">|</span> <span class="n">Metapod</span> <span class="o">|</span> <span class="mi">20</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Butterfree</span> <span class="o">|</span> <span class="mi">45</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Weedle</span> <span class="o">|</span> <span class="mi">35</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Kakuna</span> <span class="o">|</span> <span class="mi">25</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Beedrill</span> <span class="o">|</span> <span class="mi">90</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">BeedrillMega</span> <span class="n">Beedrill</span> <span class="o">|</span> <span class="mi">150</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Paras</span> <span class="o">|</span> <span class="mi">70</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Parasect</span> <span class="o">|</span> <span class="mi">95</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Venonat</span> <span class="o">|</span> <span class="mi">55</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">|</span> <span class="n">Venomoth</span> <span class="o">|</span> <span class="mi">65</span> <span class="o">|</span> <span class="n">Bug</span> <span class="o">|</span> <span class="mf">76.42857142857143</span> <span class="o">|</span> +<span class="o">+-----------------------+--------+--------+-------------------+</span> +<span class="n">Data</span> <span class="n">truncated</span><span class="o">.</span> </pre></div> </div> </section>
diff --git a/user-guide/configuration.html b/user-guide/configuration.html index 257958d..b29bfff 100644 --- a/user-guide/configuration.html +++ b/user-guide/configuration.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/data-sources.html b/user-guide/data-sources.html index 6fb9566..64be39f 100644 --- a/user-guide/data-sources.html +++ b/user-guide/data-sources.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/introduction.html b/user-guide/introduction.html index b2c7a90..64f7137 100644 --- a/user-guide/introduction.html +++ b/user-guide/introduction.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a> @@ -400,7 +415,7 @@ <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="n">In</span> <span class="p">[</span><span class="mi">1</span><span class="p">]:</span> <span class="kn">import</span><span class="w"> </span><span class="nn">datafusion</span> <span class="n">In</span> <span class="p">[</span><span class="mi">2</span><span class="p">]:</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">__version__</span> -<span class="n">Out</span><span class="p">[</span><span class="mi">2</span><span class="p">]:</span> <span class="s1">'45.2.0'</span> +<span class="n">Out</span><span class="p">[</span><span class="mi">2</span><span class="p">]:</span> <span class="s1">'46.0.0'</span> </pre></div> </div> <p>In this documentation we will also show some examples for how DataFusion integrates
diff --git a/user-guide/io/arrow.html b/user-guide/io/arrow.html index 4fd6b96..caa0e63 100644 --- a/user-guide/io/arrow.html +++ b/user-guide/io/arrow.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/avro.html b/user-guide/io/avro.html index 1f33f29..9c18f51 100644 --- a/user-guide/io/avro.html +++ b/user-guide/io/avro.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/csv.html b/user-guide/io/csv.html index d99d2f0..880ce77 100644 --- a/user-guide/io/csv.html +++ b/user-guide/io/csv.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/index.html b/user-guide/io/index.html index 30b0670..b42512d 100644 --- a/user-guide/io/index.html +++ b/user-guide/io/index.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/json.html b/user-guide/io/json.html index 63441b6..52d5377 100644 --- a/user-guide/io/json.html +++ b/user-guide/io/json.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/parquet.html b/user-guide/io/parquet.html index 00f0aa1..71d0338 100644 --- a/user-guide/io/parquet.html +++ b/user-guide/io/parquet.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/io/table_provider.html b/user-guide/io/table_provider.html index 9558846..410623c 100644 --- a/user-guide/io/table_provider.html +++ b/user-guide/io/table_provider.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="../common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="../common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>
diff --git a/user-guide/sql.html b/user-guide/sql.html index e1fe06d..e8d46a5 100644 --- a/user-guide/sql.html +++ b/user-guide/sql.html
@@ -128,6 +128,11 @@ </label> <ul> <li class="toctree-l2"> + <a class="reference internal" href="common-operations/views.html"> + Registering Views + </a> + </li> + <li class="toctree-l2"> <a class="reference internal" href="common-operations/basic-info.html"> Basic Operations </a> @@ -233,6 +238,11 @@ Introduction </a> </li> + <li class="toctree-l1"> + <a class="reference internal" href="../contributor-guide/ffi.html"> + Python Extensions + </a> + </li> </ul> <p aria-level="2" class="caption" role="heading"> <span class="caption-text"> @@ -308,6 +318,11 @@ </ul> </li> <li class="toctree-l3"> + <a class="reference internal" href="../autoapi/datafusion/io/index.html"> + datafusion.io + </a> + </li> + <li class="toctree-l3"> <a class="reference internal" href="../autoapi/datafusion/object_store/index.html"> datafusion.object_store </a>