PROTON-2297 Fix issue with composite buffer utf8 decode
Handle multi-byte encoding that cross composite array boundaries and add
some tests.
diff --git a/proton-j/src/main/java/org/apache/qpid/proton/codec/CompositeReadableBuffer.java b/proton-j/src/main/java/org/apache/qpid/proton/codec/CompositeReadableBuffer.java
index abbb40b..a4c6de9 100644
--- a/proton-j/src/main/java/org/apache/qpid/proton/codec/CompositeReadableBuffer.java
+++ b/proton-j/src/main/java/org/apache/qpid/proton/codec/CompositeReadableBuffer.java
@@ -578,12 +578,27 @@
do {
boolean endOfInput = processed == viewSpan;
step = decoder.decode(wrapper, decoded, endOfInput);
- if (step.isUnderflow() && endOfInput) {
- step = decoder.flush(decoded);
- break;
- }
- if (step.isOverflow()) {
+ if (step.isUnderflow()) {
+ if (endOfInput) {
+ step = decoder.flush(decoded);
+ break;
+ } if (wrapper.hasRemaining()) {
+ final int unprocessed = wrapper.remaining();
+ final byte[] next = contents.get(++arrayIndex);
+ final ByteBuffer previous = wrapper;
+ wrapper = ByteBuffer.allocate(unprocessed + next.length);
+ wrapper.put(previous);
+ wrapper.put(next);
+ processed += wrapper.position() - unprocessed;
+ wrapper.flip();
+ } else {
+ final byte[] next = contents.get(++arrayIndex);
+ final int wrapSize = Math.min(next.length, viewSpan - processed);
+ wrapper = ByteBuffer.wrap(next, 0, wrapSize);
+ processed += wrapSize;
+ }
+ } else if (step.isOverflow()) {
size = 2 * size + 1;
CharBuffer upsized = CharBuffer.allocate(size);
decoded.flip();
@@ -591,11 +606,6 @@
decoded = upsized;
continue;
}
-
- byte[] next = contents.get(++arrayIndex);
- int wrapSize = Math.min(next.length, viewSpan - processed);
- wrapper = ByteBuffer.wrap(next, 0, wrapSize);
- processed += wrapSize;
} while (!step.isError());
if (step.isError()) {
diff --git a/proton-j/src/test/java/org/apache/qpid/proton/codec/CompositeReadableBufferTest.java b/proton-j/src/test/java/org/apache/qpid/proton/codec/CompositeReadableBufferTest.java
index 9dc5c1b..6ae4592 100644
--- a/proton-j/src/test/java/org/apache/qpid/proton/codec/CompositeReadableBufferTest.java
+++ b/proton-j/src/test/java/org/apache/qpid/proton/codec/CompositeReadableBufferTest.java
@@ -26,6 +26,7 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.nio.InvalidMarkException;
@@ -3346,6 +3347,76 @@
assertEquals("T", buffer.readString(StandardCharsets.UTF_8.newDecoder()));
}
+ @Test
+ public void testReadUnicodeStringAcrossArrayBoundries() throws IOException {
+ String expected = "\u1f4a9\\u1f4a9\\u1f4a9";
+
+ byte[] utf8 = expected.getBytes(StandardCharsets.UTF_8);
+
+ byte[] slice1 = new byte[] { utf8[0] };
+ byte[] slice2 = new byte[utf8.length - 1];
+
+ System.arraycopy(utf8, 1, slice2, 0, slice2.length);
+
+ CompositeReadableBuffer composite = new CompositeReadableBuffer();
+ composite.append(slice1);
+ composite.append(slice2);
+
+ String result = composite.readUTF8();
+
+ assertEquals("Failed to round trip String correctly: ", expected, result);
+ }
+
+ @Test
+ public void testReadUnicodeStringAcrossMultipleArrayBoundries() throws IOException {
+ String expected = "\u1f4a9\\u1f4a9\\u1f4a9";
+
+ byte[] utf8 = expected.getBytes(StandardCharsets.UTF_8);
+
+ byte[] slice1 = new byte[] { utf8[0] };
+ byte[] slice2 = new byte[] { utf8[1], utf8[2] };
+ byte[] slice3 = new byte[] { utf8[3], utf8[4] };
+ byte[] slice4 = new byte[utf8.length - 5];
+
+ System.arraycopy(utf8, 5, slice4, 0, slice4.length);
+
+ CompositeReadableBuffer composite = new CompositeReadableBuffer();
+ composite.append(slice1);
+ composite.append(slice2);
+ composite.append(slice3);
+ composite.append(slice4);
+
+ String result = composite.readUTF8();
+
+ assertEquals("Failed to round trip String correctly: ", expected, result);
+ }
+
+ @Test
+ public void testReadUnicodeStringEachByteInOwnArray() throws IOException {
+ String expected = "\u1f4a9";
+
+ byte[] utf8 = expected.getBytes(StandardCharsets.UTF_8);
+
+ assertEquals(4, utf8.length);
+
+ byte[] slice1 = new byte[] { utf8[0] };
+ byte[] slice2 = new byte[] { utf8[1] };
+ byte[] slice3 = new byte[] { utf8[2] };
+ byte[] slice4 = new byte[] { utf8[3] };
+
+ System.arraycopy(utf8, 1, slice2, 0, slice2.length);
+
+ CompositeReadableBuffer composite = new CompositeReadableBuffer();
+ composite.append(slice1);
+ composite.append(slice2);
+ composite.append(slice3);
+ composite.append(slice4);
+
+ String result = composite.readUTF8();
+
+ assertEquals("Failed to round trip String correctly: ", expected, result);
+ }
+
//----- Tests for hashCode -----------------------------------------------//
@Test
diff --git a/proton-j/src/test/java/org/apache/qpid/proton/codec/StringTypeTest.java b/proton-j/src/test/java/org/apache/qpid/proton/codec/StringTypeTest.java
index 3bf3985..a6e6685 100644
--- a/proton-j/src/test/java/org/apache/qpid/proton/codec/StringTypeTest.java
+++ b/proton-j/src/test/java/org/apache/qpid/proton/codec/StringTypeTest.java
@@ -454,4 +454,43 @@
delegate.put(src);
}
}
+
+ @Test
+ public void testEncodeAndDecodeLargeUnicodeString() throws IOException {
+ StringBuilder unicodeStringBuilder = new StringBuilder();
+
+ unicodeStringBuilder.append((char) 1000);
+ unicodeStringBuilder.append((char) 1001);
+ unicodeStringBuilder.append((char) 1002);
+ unicodeStringBuilder.append((char) 1003);
+
+ final DecoderImpl decoder = new DecoderImpl();
+ final EncoderImpl encoder = new EncoderImpl(decoder);
+ AMQPDefinedTypes.registerAllTypes(decoder, encoder);
+ final ByteBuffer bb = ByteBuffer.allocate(1024);
+
+ final AmqpValue inputValue = new AmqpValue(unicodeStringBuilder.toString());
+ encoder.setByteBuffer(bb);
+ encoder.writeObject(inputValue);
+
+ final int size1 = bb.position() / 2;
+ final int size2 = bb.position() - size1;
+
+ final byte[] slice1 = new byte[size1];
+ final byte[] slice2 = new byte[size2];
+
+ bb.flip();
+ bb.get(slice1);
+ bb.get(slice2);
+
+ CompositeReadableBuffer composite = new CompositeReadableBuffer();
+ composite.append(slice1);
+ composite.append(slice2);
+
+ decoder.setBuffer(composite);
+
+ final AmqpValue outputValue = (AmqpValue) decoder.readObject();
+
+ assertEquals("Failed to round trip String correctly: ", unicodeStringBuilder.toString(), outputValue.getValue());
+ }
}
