sdks/python/apache_beam/coders/coder_impl.pxd

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


cimport cython

cimport cpython.ref
cimport cpython.tuple
cimport libc.stdint
cimport libc.stdlib
cimport libc.string

cdef extern from "math.h":
  libc.stdint.int64_t abs "llabs"(libc.stdint.int64_t)

ctypedef char* char_ptr

from .stream cimport InputStream, OutputStream
from apache_beam.utils cimport windowed_value


cdef object loads, dumps, create_InputStream, create_OutputStream, ByteCountingOutputStream, get_varint_size, past_unicode
# Temporarily untyped to allow monkeypatching on failed import.
#cdef type WindowedValue

cdef bint is_compiled

cdef class CoderImpl(object):
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
  cpdef decode_from_stream(self, InputStream stream, bint nested)
  cpdef bytes encode(self, value)
  cpdef decode(self, bytes encoded)
  cpdef bytes encode_nested(self, value)
  cpdef decode_nested(self, bytes encoded)
  cpdef estimate_size(self, value, bint nested=?)
  @cython.locals(varint_size=int, bits=libc.stdint.uint64_t)
  @cython.overflowcheck(False)
  cpdef int _get_nested_size(self, int inner_size, bint nested)
  cpdef get_estimated_size_and_observables(self, value, bint nested=?)


cdef class SimpleCoderImpl(CoderImpl):
  pass


cdef class StreamCoderImpl(CoderImpl):
  pass


cdef class CallbackCoderImpl(CoderImpl):
  cdef object _encoder
  cdef object _decoder
  cdef object _size_estimator


cdef object NoneType
cdef unsigned char UNKNOWN_TYPE, NONE_TYPE, INT_TYPE, FLOAT_TYPE, BOOL_TYPE
cdef unsigned char BYTES_TYPE, UNICODE_TYPE, LIST_TYPE, TUPLE_TYPE, DICT_TYPE
cdef unsigned char SET_TYPE, ITERABLE_LIKE_TYPE
cdef unsigned char PROTO_TYPE, DATACLASS_TYPE, NAMED_TUPLE_TYPE


cdef set _ITERABLE_LIKE_TYPES

cdef class FastPrimitivesCoderImpl(StreamCoderImpl):
  cdef CoderImpl fallback_coder_impl
  cdef CoderImpl iterable_coder_impl
  cdef object requires_deterministic_step_label
  cdef bint warn_deterministic_fallback
  cdef bint force_use_dill
  cdef bint use_relative_filepaths

  @cython.locals(dict_value=dict, int_value=libc.stdint.int64_t,
                 unicode_value=unicode)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
  @cython.locals(t=int)
  cpdef decode_from_stream(self, InputStream stream, bint nested)
  cdef encode_special_deterministic(self, value, OutputStream stream)
  cdef encode_type_2_67_0(self, t, OutputStream stream)
  cdef encode_type(self, t, OutputStream stream)
  cdef decode_type(self, InputStream stream)

cdef dict _pickled_types

cdef dict _unpickled_types


cdef class MapCoderImpl(StreamCoderImpl):
  cdef CoderImpl _key_coder
  cdef CoderImpl _value_coder
  cdef bint _is_deterministic


cdef class BytesCoderImpl(CoderImpl):
  pass


cdef class BooleanCoderImpl(CoderImpl):
  pass


cdef class BigEndianShortCoderImpl(StreamCoderImpl):
  pass


cdef class ByteCoderImpl(StreamCoderImpl):
  pass


cdef class SinglePrecisionFloatCoderImpl(StreamCoderImpl):
  pass


cdef class FloatCoderImpl(StreamCoderImpl):
  pass


cdef class TimestampCoderImpl(StreamCoderImpl):
  cdef object timestamp_class


cdef list small_ints
cdef class VarIntCoderImpl(StreamCoderImpl):
  @cython.locals(ivalue=libc.stdint.int64_t)
  cpdef bytes encode(self, value)


cdef class VarInt32CoderImpl(StreamCoderImpl):
  @cython.locals(ivalue=libc.stdint.int32_t)
  cpdef bytes encode(self, value)


cdef class SingletonCoderImpl(CoderImpl):
  cdef object _value


cdef class AbstractComponentCoderImpl(StreamCoderImpl):
  cdef tuple _coder_impls

  cpdef _extract_components(self, value)
  cpdef _construct_from_components(self, components)

  @cython.locals(c=CoderImpl)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
  @cython.locals(c=CoderImpl)
  cpdef decode_from_stream(self, InputStream stream, bint nested)

  @cython.locals(c=CoderImpl)
  cpdef get_estimated_size_and_observables(self, value, bint nested=?)


cdef class TupleCoderImpl(AbstractComponentCoderImpl):
  pass


cdef class SequenceCoderImpl(StreamCoderImpl):
  cdef CoderImpl _elem_coder
  cdef object _read_state
  cdef object _write_state
  cdef int _write_state_threshold

  cpdef _construct_from_sequence(self, values)

  @cython.locals(buffer=OutputStream, target_buffer_size=libc.stdint.int64_t,
                 index=libc.stdint.int64_t, prev_index=libc.stdint.int64_t)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)


cdef class TupleSequenceCoderImpl(SequenceCoderImpl):
  pass


cdef class _AbstractIterable:
  cdef object _contents


cdef class IterableCoderImpl(SequenceCoderImpl):
  cdef bint _use_abstract_iterable


cdef class ListCoderImpl(SequenceCoderImpl):
  pass


cdef object IntervalWindow

cdef class IntervalWindowCoderImpl(StreamCoderImpl):
  cdef libc.stdint.uint64_t _to_normal_time(self, libc.stdint.int64_t value)
  cdef libc.stdint.int64_t _from_normal_time(self, libc.stdint.uint64_t value)

  @cython.locals(typed_value=windowed_value._IntervalWindowBase,
                 span_millis=libc.stdint.int64_t)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)

  @cython.locals(typed_value=windowed_value._IntervalWindowBase)
  cpdef decode_from_stream(self, InputStream stream, bint nested)

  @cython.locals(typed_value=windowed_value._IntervalWindowBase,
                 span_millis=libc.stdint.int64_t)
  cpdef estimate_size(self, value, bint nested=?)


cdef int PaneInfoTiming_UNKNOWN
cdef int PaneInfoEncoding_FIRST


cdef class PaneInfoCoderImpl(StreamCoderImpl):
  cdef int _choose_encoding(self, windowed_value.PaneInfo value)

  @cython.locals(pane_info=windowed_value.PaneInfo, encoding_type=int)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)

  @cython.locals(encoded_first_byte=int, encoding_type=int)
  cpdef decode_from_stream(self, InputStream stream, bint nested)


cdef libc.stdint.uint64_t _TIME_SHIFT
cdef libc.stdint.int64_t MIN_TIMESTAMP_micros
cdef libc.stdint.int64_t MAX_TIMESTAMP_micros


cdef class _OrderedUnionCoderImpl(StreamCoderImpl):
  cdef tuple _types
  cdef tuple _coder_impls
  cdef CoderImpl _fallback_coder_impl

  @cython.locals(ix=int, c=CoderImpl)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)

  @cython.locals(ix=int, c=CoderImpl)
  cpdef decode_from_stream(self, InputStream stream, bint nested)


cdef class WindowedValueCoderImpl(StreamCoderImpl):
  """A coder for windowed values."""
  cdef CoderImpl _value_coder
  cdef CoderImpl _timestamp_coder
  cdef CoderImpl _windows_coder
  cdef CoderImpl _pane_info_coder

  cdef libc.stdint.uint64_t _to_normal_time(self, libc.stdint.int64_t value)
  cdef libc.stdint.int64_t _from_normal_time(self, libc.stdint.uint64_t value)

  @cython.locals(c=CoderImpl)
  cpdef get_estimated_size_and_observables(self, value, bint nested=?)

  @cython.locals(timestamp=libc.stdint.int64_t)
  cpdef decode_from_stream(self, InputStream stream, bint nested)

  @cython.locals(wv=windowed_value.WindowedValue, restore_sign=int)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)


cdef class ParamWindowedValueCoderImpl(WindowedValueCoderImpl):
  """A coder for windowed values with constant timestamp, windows and pane info."""
  cdef readonly libc.stdint.int64_t _timestamp
  cdef readonly object _windows
  cdef readonly windowed_value.PaneInfo _pane_info


cdef class LengthPrefixCoderImpl(StreamCoderImpl):
  cdef CoderImpl _value_coder


cdef class RowColumnEncoder:
  cdef bint encode_to_stream(self, size_t index, OutputStream stream) except -1
  cdef bint decode_from_stream(self, size_t index, InputStream stream) except -1


cdef class GenericRowColumnEncoder(RowColumnEncoder):
  cdef object column
  cdef CoderImpl coder_impl


cdef class RowCoderImpl(StreamCoderImpl):
  cdef object schema
  cdef int num_fields
  cdef list field_names
  cdef list field_nullable
  cdef object constructor
  cdef list encoding_positions
  cdef list encoding_positions_argsort
  cdef bint encoding_positions_are_trivial
  cdef list components
  cdef bint has_nullable_fields

  @cython.locals(i=int, nvals=libc.stdint.int64_t, running=int, component_coder=CoderImpl,
                 null_mask=bytes, null_mask_c=char_ptr)
  cpdef decode_from_stream(self, InputStream stream, bint nested)

  @cython.locals(i=int, nvals=libc.stdint.int64_t, k=size_t, n=size_t,
                 null_mask=bytes, null_mask_c=char_ptr)
  cpdef decode_batch_from_stream(self, dict dest, InputStream stream)

  @cython.locals(i=int, running=int, component_coder=CoderImpl)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)

  @cython.locals(i=int, k=size_t, n=size_t,
                 null_flags=libc.stdint.uint8_t[:,::1],
                 null_bits=libc.stdint.uint8_t[:,::1],
                 has_null_bits=libc.stdint.uint8_t[::1])
  cpdef encode_batch_to_stream(self, dict values, OutputStream stream)


cdef class LogicalTypeCoderImpl(StreamCoderImpl):
  cdef object logical_type
  cdef CoderImpl representation_coder

  cpdef decode_from_stream(self, InputStream stream, bint nested)
  cpdef encode_to_stream(self, value, OutputStream stream, bint nested)


cdef class BigIntegerCoderImpl(StreamCoderImpl):
  pass


cdef class DecimalCoderImpl(StreamCoderImpl):
  pass