Attempt to encode non-encoded messages (#296)
diff --git a/provider/consumer.py b/provider/consumer.py
index a061c86..12cfd75 100644
--- a/provider/consumer.py
+++ b/provider/consumer.py
@@ -469,6 +469,17 @@
return offsets
def __encodeMessageIfNeeded(self, value):
+ # let's make sure whatever data we're getting is utf-8 encoded
+ try:
+ value.decode('utf-8')
+ except UnicodeDecodeError:
+ try:
+ logging.warn('[{}] Value is not UTF-8 encoded. Attempting encoding...'.format(self.trigger))
+ value = value.encode('utf-8')
+ except UnicodeDecodeError:
+ logging.warn('[{}] Value contains non-unicode bytes. Replacing invalid bytes.'.format(self.trigger))
+ value = unicode(value, errors='replace').encode('utf-8')
+
if self.encodeValueAsJSON:
try:
parsed = json.loads(value)
