I'm trying to write a py2.7 - py3.7 compatible django-redis serializer.
I'm using django-redis==4.8.0 with django==1.11.22 and the PickleSerializer. I saw this issue https://github.com/niwinz/django-redis/pull/279 on django-redis and wrote a serializer similar to what's said in the thread. However my object seems a little bit more complex? Not sure.
My goal is to have 2 applications running at the same time, one with py2.7 and the other with py3.7. They have to be 100% compatible, and I'm not being able to get past this.
Here is the code for the serializer:
# -*- coding: utf-8 -*-
import six
from django.utils.encoding import force_bytes
from django_redis.serializers.pickle import PickleSerializer
try:
import cPickle as pickle
except ImportError:
import pickle
class CompatPickleSerializer(PickleSerializer):
def loads(self, value):
if six.PY3:
return self._loads_py3(value)
return super(CompatPickleSerializer, self).loads(force_bytes(value))
def _loads_py3(self, value):
return pickle.loads(
force_bytes(value),
fix_imports=True,
encoding='bytes'
)
Example of object I'm trying to serialize:
{
'created_at': datetime.datetime(2019, 7, 30, 20, 0, 29, 244916, tzinfo = < UTC > ),
'items': [{
'unit_price': Decimal('3.00'),
'name': 'my item',
'id': '12312312',
}]
'id': 'b5c6210d-561f-4e4e-a025-e55b39d95418',
'name': 'cart',
'customer': None,
}
The object is a lot bigger then that, but I assume that if I can do the flow with this object, I can do with a bigger one.
After trying to load the object on python 3.7.2, I get this error:
Traceback:
File "my-project/lib/python3.7/site-packages/django_redis/client/default.py" in decode
313. value = int(value)
During handling of the above exception (invalid literal for int() with base 10: b'\x80\x02}q\x01(U\tdiscountsq\x02NU\x10display_order_idq\x03NU\x12shipping_method_idq\x04NU\x0creservationsq\x05}U\ncreated_atq\x06U 2019-07-30T20:00:14.022071+00:00q\x07U\tpromocodeq\x08NU\x11shippi), another exception occurred:
File "my-project/lib/python3.7/site-packages/django/core/handlers/exception.py" in inner
41. response = get_response(request)
File "my-project/lib/python3.7/site-packages/django/core/handlers/base.py" in _get_response
187. response = self.process_exception_by_middleware(e, request)
File "my-project/lib/python3.7/site-packages/django/core/handlers/base.py" in _get_response
185. response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "my-project/lib/python3.7/site-packages/django/views/decorators/csrf.py" in wrapped_view
58. return view_func(*args, **kwargs)
File "my-project/lib/python3.7/site-packages/django/views/generic/base.py" in view
68. return self.dispatch(request, *args, **kwargs)
File "my-project/lib/python3.7/site-packages/rest_framework/views.py" in dispatch
489. response = self.handle_exception(exc)
File "my-project/lib/python3.7/site-packages/rest_framework/views.py" in handle_exception
449. self.raise_uncaught_exception(exc)
File "my-project/lib/python3.7/site-packages/rest_framework/views.py" in dispatch
486. response = handler(request, *args, **kwargs)
File "django/cart/api/cart.py" in get
98. cart = Cart.get(cart_id)
File "django/cart/models/cart.py" in get
1.190. raise e
File "django/cart/models/cart.py" in get
1.186. data = cart_cache.get(id)
File "my-project/lib/python3.7/site-packages/django_redis/cache.py" in _decorator
33. return method(self, *args, **kwargs)
File "my-project/lib/python3.7/site-packages/django_redis/cache.py" in get
82. client=client)
File "my-project/lib/python3.7/site-packages/django_redis/client/default.py" in get
208. return self.decode(value)
File "my-project/lib/python3.7/site-packages/django_redis/client/default.py" in decode
320. value = self._serializer.loads(value)
File "django/backports/django_redis/serializers.py" in loads
28. return self._loads_py3(value)
File "django/backports/django_redis/serializers.py" in _loads_py3
35. encoding='bytes'
Exception Type: TypeError at /my-url/
Exception Value: conversion from bytes to Decimal is not supported
Any ideas on what I could do?
If anyone is having this issue, I came to this solution
# -*- coding: utf-8 -*-
import six
from django_redis.serializers.pickle import PickleSerializer, pickle
class CompatPickleSerializer(PickleSerializer):
def loads(self, value):
if six.PY3:
return self._loads_py3(value)
return super(CompatPickleSerializer, self).loads(value)
def _loads_py3(self, value):
return pickle.loads(
value,
fix_imports=True,
encoding='latin1'
)
I changed from encoding='bytes' to latin1 and it worked. No need to force_bytes.
Another important thing is that you need to force the PICKLE_VERSION to 2, otherwise you won't be able to pickle the data on python2.7 if it was serialized on python3.
Hope it helps anyone.