| """ |
| New, fast version of the CloudPickler. |
| |
| This new CloudPickler class can now extend the fast C Pickler instead of the |
| previous Python implementation of the Pickler class. Because this functionality |
| is only available for Python versions 3.8+, a lot of backward-compatibility |
| code is also removed. |
| |
| Note that the C Pickler subclassing API is CPython-specific. Therefore, some |
| guards present in cloudpickle.py that were written to handle PyPy specificities |
| are not present in cloudpickle_fast.py |
| """ |
| import _collections_abc |
| import abc |
| import copyreg |
| import io |
| import itertools |
| import logging |
| import sys |
| import struct |
| import types |
| import weakref |
| import typing |
| |
| from enum import Enum |
| from collections import ChainMap, OrderedDict |
| |
| from .compat import pickle, Pickler |
| from .cloudpickle import ( |
| _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, |
| _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, |
| _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, |
| _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, |
| _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, |
| _is_parametrized_type_hint, PYPY, cell_set, |
| parametrized_type_hint_getinitargs, _create_parametrized_type_hint, |
| builtin_code_type, |
| _make_dict_keys, _make_dict_values, _make_dict_items, _make_function, |
| ) |
| |
| |
| if pickle.HIGHEST_PROTOCOL >= 5: |
| # Shorthands similar to pickle.dump/pickle.dumps |
| |
| def dump(obj, file, protocol=None, buffer_callback=None): |
| """Serialize obj as bytes streamed into file |
| |
| protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to |
| pickle.HIGHEST_PROTOCOL. This setting favors maximum communication |
| speed between processes running the same Python version. |
| |
| Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure |
| compatibility with older versions of Python. |
| """ |
| CloudPickler( |
| file, protocol=protocol, buffer_callback=buffer_callback |
| ).dump(obj) |
| |
| def dumps(obj, protocol=None, buffer_callback=None): |
| """Serialize obj as a string of bytes allocated in memory |
| |
| protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to |
| pickle.HIGHEST_PROTOCOL. This setting favors maximum communication |
| speed between processes running the same Python version. |
| |
| Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure |
| compatibility with older versions of Python. |
| """ |
| with io.BytesIO() as file: |
| cp = CloudPickler( |
| file, protocol=protocol, buffer_callback=buffer_callback |
| ) |
| cp.dump(obj) |
| return file.getvalue() |
| |
| else: |
| # Shorthands similar to pickle.dump/pickle.dumps |
| def dump(obj, file, protocol=None): |
| """Serialize obj as bytes streamed into file |
| |
| protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to |
| pickle.HIGHEST_PROTOCOL. This setting favors maximum communication |
| speed between processes running the same Python version. |
| |
| Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure |
| compatibility with older versions of Python. |
| """ |
| CloudPickler(file, protocol=protocol).dump(obj) |
| |
| def dumps(obj, protocol=None): |
| """Serialize obj as a string of bytes allocated in memory |
| |
| protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to |
| pickle.HIGHEST_PROTOCOL. This setting favors maximum communication |
| speed between processes running the same Python version. |
| |
| Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure |
| compatibility with older versions of Python. |
| """ |
| with io.BytesIO() as file: |
| cp = CloudPickler(file, protocol=protocol) |
| cp.dump(obj) |
| return file.getvalue() |
| |
| |
| load, loads = pickle.load, pickle.loads |
| |
| |
| # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS |
| # ------------------------------------------------- |
| |
| def _class_getnewargs(obj): |
| type_kwargs = {} |
| if "__module__" in obj.__dict__: |
| type_kwargs["__module__"] = obj.__module__ |
| |
| __dict__ = obj.__dict__.get('__dict__', None) |
| if isinstance(__dict__, property): |
| type_kwargs['__dict__'] = __dict__ |
| |
| return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, |
| _get_or_create_tracker_id(obj), None) |
| |
| |
| def _enum_getnewargs(obj): |
| members = {e.name: e.value for e in obj} |
| return (obj.__bases__, obj.__name__, obj.__qualname__, members, |
| obj.__module__, _get_or_create_tracker_id(obj), None) |
| |
| |
| # COLLECTION OF OBJECTS RECONSTRUCTORS |
| # ------------------------------------ |
| def _file_reconstructor(retval): |
| return retval |
| |
| |
| # COLLECTION OF OBJECTS STATE GETTERS |
| # ----------------------------------- |
| def _function_getstate(func): |
| # - Put func's dynamic attributes (stored in func.__dict__) in state. These |
| # attributes will be restored at unpickling time using |
| # f.__dict__.update(state) |
| # - Put func's members into slotstate. Such attributes will be restored at |
| # unpickling time by iterating over slotstate and calling setattr(func, |
| # slotname, slotvalue) |
| slotstate = { |
| "__name__": func.__name__, |
| "__qualname__": func.__qualname__, |
| "__annotations__": func.__annotations__, |
| "__kwdefaults__": func.__kwdefaults__, |
| "__defaults__": func.__defaults__, |
| "__module__": func.__module__, |
| "__doc__": func.__doc__, |
| "__closure__": func.__closure__, |
| } |
| |
| f_globals_ref = _extract_code_globals(func.__code__) |
| f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in |
| func.__globals__} |
| |
| closure_values = ( |
| list(map(_get_cell_contents, func.__closure__)) |
| if func.__closure__ is not None else () |
| ) |
| |
| # Extract currently-imported submodules used by func. Storing these modules |
| # in a smoke _cloudpickle_subimports attribute of the object's state will |
| # trigger the side effect of importing these modules at unpickling time |
| # (which is necessary for func to work correctly once depickled) |
| slotstate["_cloudpickle_submodules"] = _find_imported_submodules( |
| func.__code__, itertools.chain(f_globals.values(), closure_values)) |
| slotstate["__globals__"] = f_globals |
| |
| state = func.__dict__ |
| return state, slotstate |
| |
| |
| def _class_getstate(obj): |
| clsdict = _extract_class_dict(obj) |
| clsdict.pop('__weakref__', None) |
| |
| if issubclass(type(obj), abc.ABCMeta): |
| # If obj is an instance of an ABCMeta subclass, don't pickle the |
| # cache/negative caches populated during isinstance/issubclass |
| # checks, but pickle the list of registered subclasses of obj. |
| clsdict.pop('_abc_cache', None) |
| clsdict.pop('_abc_negative_cache', None) |
| clsdict.pop('_abc_negative_cache_version', None) |
| registry = clsdict.pop('_abc_registry', None) |
| if registry is None: |
| # in Python3.7+, the abc caches and registered subclasses of a |
| # class are bundled into the single _abc_impl attribute |
| clsdict.pop('_abc_impl', None) |
| (registry, _, _, _) = abc._get_dump(obj) |
| |
| clsdict["_abc_impl"] = [subclass_weakref() |
| for subclass_weakref in registry] |
| else: |
| # In the above if clause, registry is a set of weakrefs -- in |
| # this case, registry is a WeakSet |
| clsdict["_abc_impl"] = [type_ for type_ in registry] |
| |
| if "__slots__" in clsdict: |
| # pickle string length optimization: member descriptors of obj are |
| # created automatically from obj's __slots__ attribute, no need to |
| # save them in obj's state |
| if isinstance(obj.__slots__, str): |
| clsdict.pop(obj.__slots__) |
| else: |
| for k in obj.__slots__: |
| clsdict.pop(k, None) |
| |
| clsdict.pop('__dict__', None) # unpicklable property object |
| |
| return (clsdict, {}) |
| |
| |
| def _enum_getstate(obj): |
| clsdict, slotstate = _class_getstate(obj) |
| |
| members = {e.name: e.value for e in obj} |
| # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: |
| # Those attributes are already handled by the metaclass. |
| for attrname in ["_generate_next_value_", "_member_names_", |
| "_member_map_", "_member_type_", |
| "_value2member_map_"]: |
| clsdict.pop(attrname, None) |
| for member in members: |
| clsdict.pop(member) |
| # Special handling of Enum subclasses |
| return clsdict, slotstate |
| |
| |
| # COLLECTIONS OF OBJECTS REDUCERS |
| # ------------------------------- |
| # A reducer is a function taking a single argument (obj), and that returns a |
| # tuple with all the necessary data to re-construct obj. Apart from a few |
| # exceptions (list, dict, bytes, int, etc.), a reducer is necessary to |
| # correctly pickle an object. |
| # While many built-in objects (Exceptions objects, instances of the "object" |
| # class, etc), are shipped with their own built-in reducer (invoked using |
| # obj.__reduce__), some do not. The following methods were created to "fill |
| # these holes". |
| |
| def _code_reduce(obj): |
| """codeobject reducer""" |
| # If you are not sure about the order of arguments, take a look at help |
| # of the specific type from types, for example: |
| # >>> from types import CodeType |
| # >>> help(CodeType) |
| if hasattr(obj, "co_exceptiontable"): # pragma: no branch |
| # Python 3.11 and later: there are some new attributes |
| # related to the enhanced exceptions. |
| args = ( |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, |
| obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, |
| obj.co_freevars, obj.co_cellvars, |
| ) |
| elif hasattr(obj, "co_linetable"): # pragma: no branch |
| # Python 3.10 and later: obj.co_lnotab is deprecated and constructor |
| # expects obj.co_linetable instead. |
| args = ( |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, |
| obj.co_firstlineno, obj.co_linetable, obj.co_freevars, |
| obj.co_cellvars |
| ) |
| elif hasattr(obj, "co_nmeta"): # pragma: no cover |
| # "nogil" Python: modified attributes from 3.9 |
| args = ( |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_framesize, |
| obj.co_ndefaultargs, obj.co_nmeta, |
| obj.co_flags, obj.co_code, obj.co_consts, |
| obj.co_varnames, obj.co_filename, obj.co_name, |
| obj.co_firstlineno, obj.co_lnotab, obj.co_exc_handlers, |
| obj.co_jump_table, obj.co_freevars, obj.co_cellvars, |
| obj.co_free2reg, obj.co_cell2reg |
| ) |
| elif hasattr(obj, "co_posonlyargcount"): |
| # Backward compat for 3.9 and older |
| args = ( |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, |
| obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, |
| obj.co_cellvars |
| ) |
| else: |
| # Backward compat for even older versions of Python |
| args = ( |
| obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, |
| obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, |
| obj.co_names, obj.co_varnames, obj.co_filename, |
| obj.co_name, obj.co_firstlineno, obj.co_lnotab, |
| obj.co_freevars, obj.co_cellvars |
| ) |
| return types.CodeType, args |
| |
| |
| def _cell_reduce(obj): |
| """Cell (containing values of a function's free variables) reducer""" |
| try: |
| obj.cell_contents |
| except ValueError: # cell is empty |
| return _make_empty_cell, () |
| else: |
| return _make_cell, (obj.cell_contents, ) |
| |
| |
| def _classmethod_reduce(obj): |
| orig_func = obj.__func__ |
| return type(obj), (orig_func,) |
| |
| |
| def _file_reduce(obj): |
| """Save a file""" |
| import io |
| |
| if not hasattr(obj, "name") or not hasattr(obj, "mode"): |
| raise pickle.PicklingError( |
| "Cannot pickle files that do not map to an actual file" |
| ) |
| if obj is sys.stdout: |
| return getattr, (sys, "stdout") |
| if obj is sys.stderr: |
| return getattr, (sys, "stderr") |
| if obj is sys.stdin: |
| raise pickle.PicklingError("Cannot pickle standard input") |
| if obj.closed: |
| raise pickle.PicklingError("Cannot pickle closed files") |
| if hasattr(obj, "isatty") and obj.isatty(): |
| raise pickle.PicklingError( |
| "Cannot pickle files that map to tty objects" |
| ) |
| if "r" not in obj.mode and "+" not in obj.mode: |
| raise pickle.PicklingError( |
| "Cannot pickle files that are not opened for reading: %s" |
| % obj.mode |
| ) |
| |
| name = obj.name |
| |
| retval = io.StringIO() |
| |
| try: |
| # Read the whole file |
| curloc = obj.tell() |
| obj.seek(0) |
| contents = obj.read() |
| obj.seek(curloc) |
| except IOError as e: |
| raise pickle.PicklingError( |
| "Cannot pickle file %s as it cannot be read" % name |
| ) from e |
| retval.write(contents) |
| retval.seek(curloc) |
| |
| retval.name = name |
| return _file_reconstructor, (retval,) |
| |
| |
| def _getset_descriptor_reduce(obj): |
| return getattr, (obj.__objclass__, obj.__name__) |
| |
| |
| def _mappingproxy_reduce(obj): |
| return types.MappingProxyType, (dict(obj),) |
| |
| |
| def _memoryview_reduce(obj): |
| return bytes, (obj.tobytes(),) |
| |
| |
| def _module_reduce(obj): |
| if _should_pickle_by_reference(obj): |
| return subimport, (obj.__name__,) |
| else: |
| # Some external libraries can populate the "__builtins__" entry of a |
| # module's `__dict__` with unpicklable objects (see #316). For that |
| # reason, we do not attempt to pickle the "__builtins__" entry, and |
| # restore a default value for it at unpickling time. |
| state = obj.__dict__.copy() |
| state.pop('__builtins__', None) |
| return dynamic_subimport, (obj.__name__, state) |
| |
| |
| def _method_reduce(obj): |
| return (types.MethodType, (obj.__func__, obj.__self__)) |
| |
| |
| def _logger_reduce(obj): |
| return logging.getLogger, (obj.name,) |
| |
| |
| def _root_logger_reduce(obj): |
| return logging.getLogger, () |
| |
| |
| def _property_reduce(obj): |
| return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__) |
| |
| |
| def _weakset_reduce(obj): |
| return weakref.WeakSet, (list(obj),) |
| |
| |
| def _dynamic_class_reduce(obj): |
| """ |
| Save a class that can't be stored as module global. |
| |
| This method is used to serialize classes that are defined inside |
| functions, or that otherwise can't be serialized as attribute lookups |
| from global modules. |
| """ |
| if Enum is not None and issubclass(obj, Enum): |
| return ( |
| _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), |
| None, None, _class_setstate |
| ) |
| else: |
| return ( |
| _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), |
| None, None, _class_setstate |
| ) |
| |
| |
| def _class_reduce(obj): |
| """Select the reducer depending on the dynamic nature of the class obj""" |
| if obj is type(None): # noqa |
| return type, (None,) |
| elif obj is type(Ellipsis): |
| return type, (Ellipsis,) |
| elif obj is type(NotImplemented): |
| return type, (NotImplemented,) |
| elif obj in _BUILTIN_TYPE_NAMES: |
| return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) |
| elif not _should_pickle_by_reference(obj): |
| return _dynamic_class_reduce(obj) |
| return NotImplemented |
| |
| |
| def _dict_keys_reduce(obj): |
| # Safer not to ship the full dict as sending the rest might |
| # be unintended and could potentially cause leaking of |
| # sensitive information |
| return _make_dict_keys, (list(obj), ) |
| |
| |
| def _dict_values_reduce(obj): |
| # Safer not to ship the full dict as sending the rest might |
| # be unintended and could potentially cause leaking of |
| # sensitive information |
| return _make_dict_values, (list(obj), ) |
| |
| |
| def _dict_items_reduce(obj): |
| return _make_dict_items, (dict(obj), ) |
| |
| |
| def _odict_keys_reduce(obj): |
| # Safer not to ship the full dict as sending the rest might |
| # be unintended and could potentially cause leaking of |
| # sensitive information |
| return _make_dict_keys, (list(obj), True) |
| |
| |
| def _odict_values_reduce(obj): |
| # Safer not to ship the full dict as sending the rest might |
| # be unintended and could potentially cause leaking of |
| # sensitive information |
| return _make_dict_values, (list(obj), True) |
| |
| |
| def _odict_items_reduce(obj): |
| return _make_dict_items, (dict(obj), True) |
| |
| |
| # COLLECTIONS OF OBJECTS STATE SETTERS |
| # ------------------------------------ |
| # state setters are called at unpickling time, once the object is created and |
| # it has to be updated to how it was at unpickling time. |
| |
| |
| def _function_setstate(obj, state): |
| """Update the state of a dynamic function. |
| |
| As __closure__ and __globals__ are readonly attributes of a function, we |
| cannot rely on the native setstate routine of pickle.load_build, that calls |
| setattr on items of the slotstate. Instead, we have to modify them inplace. |
| """ |
| state, slotstate = state |
| obj.__dict__.update(state) |
| |
| obj_globals = slotstate.pop("__globals__") |
| obj_closure = slotstate.pop("__closure__") |
| # _cloudpickle_subimports is a set of submodules that must be loaded for |
| # the pickled function to work correctly at unpickling time. Now that these |
| # submodules are depickled (hence imported), they can be removed from the |
| # object's state (the object state only served as a reference holder to |
| # these submodules) |
| slotstate.pop("_cloudpickle_submodules") |
| |
| obj.__globals__.update(obj_globals) |
| obj.__globals__["__builtins__"] = __builtins__ |
| |
| if obj_closure is not None: |
| for i, cell in enumerate(obj_closure): |
| try: |
| value = cell.cell_contents |
| except ValueError: # cell is empty |
| continue |
| cell_set(obj.__closure__[i], value) |
| |
| for k, v in slotstate.items(): |
| setattr(obj, k, v) |
| |
| |
| def _class_setstate(obj, state): |
| state, slotstate = state |
| registry = None |
| for attrname, attr in state.items(): |
| if attrname == "_abc_impl": |
| registry = attr |
| else: |
| setattr(obj, attrname, attr) |
| if registry is not None: |
| for subclass in registry: |
| obj.register(subclass) |
| |
| return obj |
| |
| |
| class CloudPickler(Pickler): |
| # set of reducers defined and used by cloudpickle (private) |
| _dispatch_table = {} |
| _dispatch_table[classmethod] = _classmethod_reduce |
| _dispatch_table[io.TextIOWrapper] = _file_reduce |
| _dispatch_table[logging.Logger] = _logger_reduce |
| _dispatch_table[logging.RootLogger] = _root_logger_reduce |
| _dispatch_table[memoryview] = _memoryview_reduce |
| _dispatch_table[property] = _property_reduce |
| _dispatch_table[staticmethod] = _classmethod_reduce |
| _dispatch_table[CellType] = _cell_reduce |
| _dispatch_table[types.CodeType] = _code_reduce |
| _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce |
| _dispatch_table[types.ModuleType] = _module_reduce |
| _dispatch_table[types.MethodType] = _method_reduce |
| _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce |
| _dispatch_table[weakref.WeakSet] = _weakset_reduce |
| _dispatch_table[typing.TypeVar] = _typevar_reduce |
| _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce |
| _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce |
| _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce |
| _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce |
| _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce |
| _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce |
| _dispatch_table[abc.abstractmethod] = _classmethod_reduce |
| _dispatch_table[abc.abstractclassmethod] = _classmethod_reduce |
| _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce |
| _dispatch_table[abc.abstractproperty] = _property_reduce |
| |
| dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) |
| |
| # function reducers are defined as instance methods of CloudPickler |
| # objects, as they rely on a CloudPickler attribute (globals_ref) |
| def _dynamic_function_reduce(self, func): |
| """Reduce a function that is not pickleable via attribute lookup.""" |
| newargs = self._function_getnewargs(func) |
| state = _function_getstate(func) |
| return (_make_function, newargs, state, None, None, |
| _function_setstate) |
| |
| def _function_reduce(self, obj): |
| """Reducer for function objects. |
| |
| If obj is a top-level attribute of a file-backed module, this |
| reducer returns NotImplemented, making the CloudPickler fallback to |
| traditional _pickle.Pickler routines to save obj. Otherwise, it reduces |
| obj using a custom cloudpickle reducer designed specifically to handle |
| dynamic functions. |
| |
| As opposed to cloudpickle.py, There no special handling for builtin |
| pypy functions because cloudpickle_fast is CPython-specific. |
| """ |
| if _should_pickle_by_reference(obj): |
| return NotImplemented |
| else: |
| return self._dynamic_function_reduce(obj) |
| |
| def _function_getnewargs(self, func): |
| code = func.__code__ |
| |
| # base_globals represents the future global namespace of func at |
| # unpickling time. Looking it up and storing it in |
| # CloudpiPickler.globals_ref allow functions sharing the same globals |
| # at pickling time to also share them once unpickled, at one condition: |
| # since globals_ref is an attribute of a CloudPickler instance, and |
| # that a new CloudPickler is created each time pickle.dump or |
| # pickle.dumps is called, functions also need to be saved within the |
| # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: |
| # cloudpickle.dumps([f1, f2])). There is no such limitation when using |
| # CloudPickler.dump, as long as the multiple invocations are bound to |
| # the same CloudPickler. |
| base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) |
| |
| if base_globals == {}: |
| # Add module attributes used to resolve relative imports |
| # instructions inside func. |
| for k in ["__package__", "__name__", "__path__", "__file__"]: |
| if k in func.__globals__: |
| base_globals[k] = func.__globals__[k] |
| |
| # Do not bind the free variables before the function is created to |
| # avoid infinite recursion. |
| if func.__closure__ is None: |
| closure = None |
| else: |
| closure = tuple( |
| _make_empty_cell() for _ in range(len(code.co_freevars))) |
| |
| return code, base_globals, None, None, closure |
| |
| def dump(self, obj): |
| try: |
| return Pickler.dump(self, obj) |
| except RuntimeError as e: |
| if "recursion" in e.args[0]: |
| msg = ( |
| "Could not pickle object as excessively deep recursion " |
| "required." |
| ) |
| raise pickle.PicklingError(msg) from e |
| else: |
| raise |
| |
| if pickle.HIGHEST_PROTOCOL >= 5: |
| def __init__(self, file, protocol=None, buffer_callback=None): |
| if protocol is None: |
| protocol = DEFAULT_PROTOCOL |
| Pickler.__init__( |
| self, file, protocol=protocol, buffer_callback=buffer_callback |
| ) |
| # map functions __globals__ attribute ids, to ensure that functions |
| # sharing the same global namespace at pickling time also share |
| # their global namespace at unpickling time. |
| self.globals_ref = {} |
| self.proto = int(protocol) |
| else: |
| def __init__(self, file, protocol=None): |
| if protocol is None: |
| protocol = DEFAULT_PROTOCOL |
| Pickler.__init__(self, file, protocol=protocol) |
| # map functions __globals__ attribute ids, to ensure that functions |
| # sharing the same global namespace at pickling time also share |
| # their global namespace at unpickling time. |
| self.globals_ref = {} |
| assert hasattr(self, 'proto') |
| |
| if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: |
| # Pickler is the C implementation of the CPython pickler and therefore |
| # we rely on reduce_override method to customize the pickler behavior. |
| |
| # `CloudPickler.dispatch` is only left for backward compatibility - note |
| # that when using protocol 5, `CloudPickler.dispatch` is not an |
| # extension of `Pickler.dispatch` dictionary, because CloudPickler |
| # subclasses the C-implemented Pickler, which does not expose a |
| # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler |
| # used `CloudPickler.dispatch` as a class-level attribute storing all |
| # reducers implemented by cloudpickle, but the attribute name was not a |
| # great choice given the meaning of `CloudPickler.dispatch` when |
| # `CloudPickler` extends the pure-python pickler. |
| dispatch = dispatch_table |
| |
| # Implementation of the reducer_override callback, in order to |
| # efficiently serialize dynamic functions and classes by subclassing |
| # the C-implemented Pickler. |
| # TODO: decorrelate reducer_override (which is tied to CPython's |
| # implementation - would it make sense to backport it to pypy? - and |
| # pickle's protocol 5 which is implementation agnostic. Currently, the |
| # availability of both notions coincide on CPython's pickle and the |
| # pickle5 backport, but it may not be the case anymore when pypy |
| # implements protocol 5 |
| |
| def reducer_override(self, obj): |
| """Type-agnostic reducing callback for function and classes. |
| |
| For performance reasons, subclasses of the C _pickle.Pickler class |
| cannot register custom reducers for functions and classes in the |
| dispatch_table. Reducer for such types must instead implemented in |
| the special reducer_override method. |
| |
| Note that method will be called for any object except a few |
| builtin-types (int, lists, dicts etc.), which differs from reducers |
| in the Pickler's dispatch_table, each of them being invoked for |
| objects of a specific type only. |
| |
| This property comes in handy for classes: although most classes are |
| instances of the ``type`` metaclass, some of them can be instances |
| of other custom metaclasses (such as enum.EnumMeta for example). In |
| particular, the metaclass will likely not be known in advance, and |
| thus cannot be special-cased using an entry in the dispatch_table. |
| reducer_override, among other things, allows us to register a |
| reducer that will be called for any class, independently of its |
| type. |
| |
| |
| Notes: |
| |
| * reducer_override has the priority over dispatch_table-registered |
| reducers. |
| * reducer_override can be used to fix other limitations of |
| cloudpickle for other types that suffered from type-specific |
| reducers, such as Exceptions. See |
| https://github.com/cloudpipe/cloudpickle/issues/248 |
| """ |
| if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch |
| return ( |
| _create_parametrized_type_hint, |
| parametrized_type_hint_getinitargs(obj) |
| ) |
| t = type(obj) |
| try: |
| is_anyclass = issubclass(t, type) |
| except TypeError: # t is not a class (old Boost; see SF #502085) |
| is_anyclass = False |
| |
| if is_anyclass: |
| return _class_reduce(obj) |
| elif isinstance(obj, types.FunctionType): |
| return self._function_reduce(obj) |
| else: |
| # fallback to save_global, including the Pickler's |
| # dispatch_table |
| return NotImplemented |
| |
| else: |
| # When reducer_override is not available, hack the pure-Python |
| # Pickler's types.FunctionType and type savers. Note: the type saver |
| # must override Pickler.save_global, because pickle.py contains a |
| # hard-coded call to save_global when pickling meta-classes. |
| dispatch = Pickler.dispatch.copy() |
| |
| def _save_reduce_pickle5(self, func, args, state=None, listitems=None, |
| dictitems=None, state_setter=None, obj=None): |
| save = self.save |
| write = self.write |
| self.save_reduce( |
| func, args, state=None, listitems=listitems, |
| dictitems=dictitems, obj=obj |
| ) |
| # backport of the Python 3.8 state_setter pickle operations |
| save(state_setter) |
| save(obj) # simple BINGET opcode as obj is already memoized. |
| save(state) |
| write(pickle.TUPLE2) |
| # Trigger a state_setter(obj, state) function call. |
| write(pickle.REDUCE) |
| # The purpose of state_setter is to carry-out an |
| # inplace modification of obj. We do not care about what the |
| # method might return, so its output is eventually removed from |
| # the stack. |
| write(pickle.POP) |
| |
| def save_global(self, obj, name=None, pack=struct.pack): |
| """ |
| Save a "global". |
| |
| The name of this method is somewhat misleading: all types get |
| dispatched here. |
| """ |
| if obj is type(None): # noqa |
| return self.save_reduce(type, (None,), obj=obj) |
| elif obj is type(Ellipsis): |
| return self.save_reduce(type, (Ellipsis,), obj=obj) |
| elif obj is type(NotImplemented): |
| return self.save_reduce(type, (NotImplemented,), obj=obj) |
| elif obj in _BUILTIN_TYPE_NAMES: |
| return self.save_reduce( |
| _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) |
| |
| if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch |
| # Parametrized typing constructs in Python < 3.7 are not |
| # compatible with type checks and ``isinstance`` semantics. For |
| # this reason, it is easier to detect them using a |
| # duck-typing-based check (``_is_parametrized_type_hint``) than |
| # to populate the Pickler's dispatch with type-specific savers. |
| self.save_reduce( |
| _create_parametrized_type_hint, |
| parametrized_type_hint_getinitargs(obj), |
| obj=obj |
| ) |
| elif name is not None: |
| Pickler.save_global(self, obj, name=name) |
| elif not _should_pickle_by_reference(obj, name=name): |
| self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) |
| else: |
| Pickler.save_global(self, obj, name=name) |
| dispatch[type] = save_global |
| |
| def save_function(self, obj, name=None): |
| """ Registered with the dispatch to handle all function types. |
| |
| Determines what kind of function obj is (e.g. lambda, defined at |
| interactive prompt, etc) and handles the pickling appropriately. |
| """ |
| if _should_pickle_by_reference(obj, name=name): |
| return Pickler.save_global(self, obj, name=name) |
| elif PYPY and isinstance(obj.__code__, builtin_code_type): |
| return self.save_pypy_builtin_func(obj) |
| else: |
| return self._save_reduce_pickle5( |
| *self._dynamic_function_reduce(obj), obj=obj |
| ) |
| |
| def save_pypy_builtin_func(self, obj): |
| """Save pypy equivalent of builtin functions. |
| PyPy does not have the concept of builtin-functions. Instead, |
| builtin-functions are simple function instances, but with a |
| builtin-code attribute. |
| Most of the time, builtin functions should be pickled by attribute. |
| But PyPy has flaky support for __qualname__, so some builtin |
| functions such as float.__new__ will be classified as dynamic. For |
| this reason only, we created this special routine. Because |
| builtin-functions are not expected to have closure or globals, |
| there is no additional hack (compared the one already implemented |
| in pickle) to protect ourselves from reference cycles. A simple |
| (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note |
| also that PyPy improved their support for __qualname__ in v3.6, so |
| this routing should be removed when cloudpickle supports only PyPy |
| 3.6 and later. |
| """ |
| rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, |
| obj.__defaults__, obj.__closure__), |
| obj.__dict__) |
| self.save_reduce(*rv, obj=obj) |
| |
| dispatch[types.FunctionType] = save_function |
