PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces wrong results in Tez (rohini)
git-svn-id: https://svn.apache.org/repos/asf/pig/branches/branch-0.15@1746360 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index b602e26..2711778 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -28,6 +28,8 @@
BUG FIXES
+PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces wrong results in Tez (rohini)
+
PIG-4860: Loading data using OrcStorage() accepts only default FileSystem path (beriaanirudh via rohini)
PIG-4867: -stop_on_failure does not work with Tez (rohini)
diff --git a/src/org/apache/pig/data/utils/SedesHelper.java b/src/org/apache/pig/data/utils/SedesHelper.java
index 1a73a77..f0be7c1 100644
--- a/src/org/apache/pig/data/utils/SedesHelper.java
+++ b/src/org/apache/pig/data/utils/SedesHelper.java
@@ -61,25 +61,25 @@
public static void writeChararray(DataOutput out, String s) throws IOException {
// a char can take up to 3 bytes in the modified utf8 encoding
// used by DataOutput.writeUTF, so use UNSIGNED_SHORT_MAX/3
- if (s.length() < BinInterSedes.UNSIGNED_SHORT_MAX / 3) {
+ byte[] utfBytes = s.getBytes(BinInterSedes.UTF8);
+ int length = utfBytes.length;
+ if (length < BinInterSedes.UNSIGNED_SHORT_MAX) {
out.writeByte(BinInterSedes.SMALLCHARARRAY);
- out.writeUTF(s);
+ out.writeShort(length);
} else {
- byte[] utfBytes = s.getBytes(BinInterSedes.UTF8);
- int length = utfBytes.length;
-
out.writeByte(BinInterSedes.CHARARRAY);
out.writeInt(length);
- out.write(utfBytes);
}
+ out.write(utfBytes);
}
public static String readChararray(DataInput in, byte type) throws IOException {
+ int size;
if (type == BinInterSedes.SMALLCHARARRAY) {
- return in.readUTF();
+ size = in.readUnsignedShort();
+ } else {
+ size = in.readInt();
}
-
- int size = in.readInt();
byte[] buf = new byte[size];
in.readFully(buf);
return new String(buf, BinInterSedes.UTF8);
