IGNITE-13863: Fix Null reading and writing
This closes #6
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index ad2a770..6860583 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -20,11 +20,13 @@
from pyignite.constants import *
from pyignite.exceptions import ParseError
+
from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -68,8 +70,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -91,7 +98,10 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -102,6 +112,9 @@
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -150,8 +163,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
@@ -243,8 +261,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -266,7 +289,10 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -277,6 +303,9 @@
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -330,8 +359,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -420,12 +454,18 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.type, super().to_python(
+ obj_type = getattr(ctype_object, "type", None)
+ if obj_type is None:
+ return None
+ return obj_type, super().to_python(
ctype_object, *args, **kwargs
)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
return super().from_python(value, type_id)
@@ -539,9 +579,13 @@
@classmethod
def parse(cls, client: 'Client'):
from pyignite.datatypes import Struct
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
# ignore full schema, always retrieve fields' types and order
@@ -572,14 +616,17 @@
@classmethod
def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs):
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
if not client:
raise ParseError(
- 'Can not query binary type {}'.format(ctype_object.type_id)
+ 'Can not query binary type {}'.format(type_id)
)
data_class = client.query_binary_type(
- ctype_object.type_id,
+ type_id,
ctype_object.schema_id
)
result = data_class()
@@ -596,6 +643,8 @@
@classmethod
def from_python(cls, value: object):
+ if value is None:
+ return Null.from_python()
if getattr(value, '_buffer', None) is None:
client = cls.find_client()
diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py
index 9f23ec6..23b9cc4 100644
--- a/pyignite/datatypes/internal.py
+++ b/pyignite/datatypes/internal.py
@@ -479,7 +479,10 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
super().to_python(
getattr(ctype_object, 'element_{}'.format(i)),
diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py
index 3763b96..1b41728 100644
--- a/pyignite/datatypes/primitive_arrays.py
+++ b/pyignite/datatypes/primitive_arrays.py
@@ -17,6 +17,7 @@
from typing import Any
from pyignite.constants import *
+from . import Null
from .base import IgniteDataType
from .primitive import *
from .type_codes import *
@@ -61,8 +62,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
cls.__name__,
@@ -82,12 +88,18 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(ctype_object.data[i])
return result
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -112,7 +124,10 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return bytearray(ctype_object.data)
+ data = getattr(ctype_object, "data", None)
+ if data is None:
+ return None
+ return bytearray(data)
@classmethod
def from_python(cls, value):
@@ -210,6 +225,9 @@
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
header.type_code = int.from_bytes(
@@ -282,6 +300,8 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
values = super().to_python(ctype_object, *args, **kwargs)
+ if values is None:
+ return None
return [
v.to_bytes(
ctypes.sizeof(cls.primitive_type.c_type),
@@ -302,7 +322,10 @@
def to_python(cls, ctype_object, *args, **kwargs):
if not ctype_object:
return None
- result = [False] * ctype_object.length
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ result = [False] * length
+ for i in range(length):
result[i] = ctype_object.data[i] != 0
return result
diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py
index 033ac9e..53f12d2 100644
--- a/pyignite/datatypes/primitive_objects.py
+++ b/pyignite/datatypes/primitive_objects.py
@@ -17,10 +17,12 @@
from pyignite.constants import *
from pyignite.utils import unsigned
+
from .base import IgniteDataType
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -60,16 +62,21 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
data_type = cls.build_c_type()
- buffer = client.recv(ctypes.sizeof(data_type))
+ buffer = tc_type + client.recv(ctypes.sizeof(data_type) - len(tc_type))
return data_type, buffer
@staticmethod
def to_python(ctype_object, *args, **kwargs):
- return ctype_object.value
+ return getattr(ctype_object, "value", None)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -185,13 +192,18 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value.to_bytes(
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value.to_bytes(
ctypes.sizeof(cls.c_type),
byteorder=PROTOCOL_BYTE_ORDER
).decode(PROTOCOL_CHAR_ENCODING)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
if type(value) is str:
value = value.encode(PROTOCOL_CHAR_ENCODING)
# assuming either a bytes or an integer
@@ -218,5 +230,8 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value != 0
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value != 0
diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py
index c65cae4..0f16735 100644
--- a/pyignite/datatypes/standard.py
+++ b/pyignite/datatypes/standard.py
@@ -276,8 +276,6 @@
UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
- UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
-
@staticmethod
def hashcode(value: 'UUID', *args, **kwargs) -> int:
msb = value.int >> 64
@@ -303,6 +301,9 @@
@classmethod
def from_python(cls, value: uuid.UUID):
+ if value is None:
+ return Null.from_python()
+
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -548,8 +549,6 @@
cls.type_code,
byteorder=PROTOCOL_BYTE_ORDER
)
- if value is None:
- return Null.from_python(value)
data_object.type_id, data_object.ordinal = value
return bytes(data_object)
@@ -601,8 +600,13 @@
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
for i in range(header.length):
@@ -623,7 +627,10 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
cls.standard_type.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -634,6 +641,8 @@
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -796,6 +805,9 @@
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
header_class = cls.build_header_class()
header = header_class()
@@ -815,7 +827,9 @@
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- type_id = ctype_object.type_id
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
return type_id, super().to_python(ctype_object, *args, **kwargs)
diff --git a/tests/test_binary.py b/tests/test_binary.py
index 4c45afb..46554ea 100644
--- a/tests/test_binary.py
+++ b/tests/test_binary.py
@@ -12,14 +12,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import re
from collections import OrderedDict
from decimal import Decimal
from pyignite import GenericObjectMeta
from pyignite.datatypes import (
- BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String,
-)
+ BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String, ByteObject, ShortObject, FloatObject,
+ DoubleObject, CharObject, UUIDObject, DateObject, TimestampObject, TimeObject, EnumObject, BinaryEnumObject,
+ ByteArrayObject, ShortArrayObject, IntArrayObject, LongArrayObject, FloatArrayObject, DoubleArrayObject,
+ CharArrayObject, BoolArrayObject, UUIDArrayObject, DateArrayObject, TimestampArrayObject, TimeArrayObject,
+ EnumArrayObject, StringArrayObject, DecimalArrayObject, ObjectArrayObject, CollectionObject, MapObject)
from pyignite.datatypes.prop_codes import *
@@ -308,8 +311,8 @@
def test_complex_object_hash(client):
"""
- Test that Python client correctly calculates hash of the binary
- object that contains negative bytes.
+ Test that Python client correctly calculates hash of the binary object that
+ contains negative bytes.
"""
class Internal(
metaclass=GenericObjectMeta,
@@ -355,3 +358,36 @@
hash_utf8 = BinaryObject.hashcode(obj_utf8, client=client)
assert hash_utf8 == -1945378474, 'Invalid hashcode value for object with UTF-8 strings'
+
+
+def test_complex_object_null_fields(client):
+ """
+ Test that Python client can correctly write and read binary object that
+ contains null fields.
+ """
+ def camel_to_snake(name):
+ return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
+
+ fields = {camel_to_snake(type_.__name__): type_ for type_ in [
+ ByteObject, ShortObject, IntObject, LongObject, FloatObject, DoubleObject, CharObject, BoolObject, UUIDObject,
+ DateObject, TimestampObject, TimeObject, EnumObject, BinaryEnumObject, ByteArrayObject, ShortArrayObject,
+ IntArrayObject, LongArrayObject, FloatArrayObject, DoubleArrayObject, CharArrayObject, BoolArrayObject,
+ UUIDArrayObject, DateArrayObject, TimestampArrayObject, TimeArrayObject, EnumArrayObject, String,
+ StringArrayObject, DecimalObject, DecimalArrayObject, ObjectArrayObject, CollectionObject, MapObject,
+ BinaryObject]}
+
+ class AllTypesObject(metaclass=GenericObjectMeta, type_name='AllTypesObject', schema=fields):
+ pass
+
+ key = 42
+ null_fields_value = AllTypesObject()
+
+ for field in fields.keys():
+ setattr(null_fields_value, field, None)
+
+ cache = client.get_or_create_cache('all_types_test_cache')
+ cache.put(key, null_fields_value)
+
+ got_obj = cache.get(key)
+
+ assert got_obj == null_fields_value, 'Objects mismatch'
