test/org/apache/pig/test/TestDataModel.java - pig - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements. See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership. The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License. You may obtain a copy of the License at
  * http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.pig.test;

 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;

 import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;

 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.InternalMap;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.junit.Test;

 /**
  * This class will exercise the basic Pig data model and members. It tests for proper behavior in
  * assigment and comparision, as well as function application.
  */
 public class TestDataModel {

     @Test
     public void testDatum() throws Exception {
         // Datum is an empty class, but in the event we
         // ever add a method, test it here
     }

     @Test
     public void testTuple() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         int arity = 5;
         int[] input1 = { 1, 2, 3, 4, 5 };
         String[] input2 = { "1", "2", "3", "4", "5" };
         String[] input3 = { "1", "2", "3", "4", "5", "6" };

         // validate construction and equality
         Tuple f1 = Util.loadFlatTuple(tf.newTuple(arity), input1);
         Tuple f2 = Util.loadFlatTuple(tf.newTuple(arity), input1);
         Tuple f3 = tf.newTuple(arity);
         assertEquals(arity, f1.size());
         assertEquals(f1, f2);

         // invalid equality
         f2 = Util.loadTuple(tf.newTuple(input3.length), input3);
         assertFalse(f1.equals(f2));

         // copy equality
         /*
          * f2.copyFrom(f1);
          * assertTrue(f1.equals(f2));
          */

         // append function and equality
         int[] input4 = { 1, 2, 3 };
         int[] input5 = { 4, 5 };
         /*
          * f1 = Util.loadFlatTuple(tf.newTuple(input4.length), input4);
          * f2 = Util.loadFlatTuple(tf.newTuple(input5.length), input5);
          * f3 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
          * f1.appendTuple(f2);
          * assertTrue(f3.equals(f1));
          */

         // arity then value comparision behavior
         f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1); // 1,2,3,4,5
         f2 = Util.loadFlatTuple(tf.newTuple(input4.length), input4); // 1,2,3
         assertTrue(f1.compareTo(f2) > 0);
         assertFalse(f1.compareTo(f2) < 0);

         int[] input6 = { 1, 2, 3, 4, 6 };
         f2 = Util.loadFlatTuple(tf.newTuple(input6.length), input6);
         assertTrue(f1.compareTo(f2) < 0);
         assertFalse(f1.compareTo(f2) > 0);

         // delimited export
         String expected = "1:2:3:4:5";
         f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
         assertTrue(expected.equals(f1.toDelimitedString(":")));

         // value read / write & marshalling
         PipedOutputStream pos = new PipedOutputStream();
         DataOutputStream dos = new DataOutputStream(pos);
         PipedInputStream pis = new PipedInputStream(pos);
         DataInputStream dis = new DataInputStream(pis);
         f1.write(dos);
         f1.write(dos);
         f2.readFields(dis);
         assertTrue(f1.equals(f2));
         f2.readFields(dis);
         assertTrue(f1.equals(f2));
     }

     @Test
     public void testNestTuple() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         int[][] input1 = { { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 },
                            { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 } };
         int[][] input2 = { { 1, 2 }, { 1, 2 } };

         Tuple n1 = Util.loadNestTuple(tf.newTuple(input1.length), input1);
         Tuple n2 = tf.newTuple();

         n2 = Util.loadNestTuple(tf.newTuple(input2.length), input2);
     }

     @Test
     public void testReadWrite() throws Exception {
         // Create a tuple with every data type in it, and then read and
         // write it, both via DataReaderWriter and Tuple.readFields
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = giveMeOneOfEach();

         File file = File.createTempFile("Tuple", "put");
         FileOutputStream fos = new FileOutputStream(file);
         DataOutput out = new DataOutputStream(fos);
         t1.write(out);
         t1.write(out); // twice in a row on purpose
         fos.close();

         FileInputStream fis = new FileInputStream(file);
         DataInput in = new DataInputStream(fis);
         for (int i = 0; i < 2; i++) {
             Tuple after = tf.newTuple();
             after.readFields(in);

             Object o = after.get(0);
             assertTrue("isa Tuple", o instanceof Tuple);
             Tuple t3 = (Tuple)o;
             o = t3.get(0);
             assertTrue("isa Integer", o instanceof Integer);
             assertEquals(new Integer(3), (Integer)o);
             o = t3.get(1);
             assertTrue("isa Float", o instanceof Float);
             assertEquals(new Float(3.0), (Float)o);

             o = after.get(1);
             assertTrue("isa Bag", o instanceof DataBag);
             DataBag b = (DataBag)o;
             Iterator<Tuple> j = b.iterator();
             Tuple[] ts = new Tuple[2];
             assertTrue("first tuple in bag", j.hasNext());
             ts[0] = j.next();
             assertTrue("second tuple in bag", j.hasNext());
             ts[1] = j.next();
             o = ts[0].get(0);
             assertTrue("isa Integer", o instanceof Integer);
             assertEquals(new Integer(4), (Integer)o);
             o = ts[1].get(0);
             assertTrue("isa String", o instanceof String);
             assertEquals("mary had a little lamb", (String)o);

             o = after.get(2);
             assertTrue("isa Map", o instanceof Map);
             Map<String, Object> m = (Map<String, Object>)o;
             assertEquals("world", (String)m.get("hello"));
             assertEquals("all", (String)m.get("goodbye"));
             assertNull(m.get("fred"));

             o = after.get(3);
             assertTrue("isa Integer", o instanceof Integer);
             Integer ii = (Integer)o;
             assertEquals(new Integer(42), ii);

             o = after.get(4);
             assertTrue("isa Long", o instanceof Long);
             Long l = (Long)o;
             assertEquals(new Long(5000000000L), l);

             o = after.get(5);
             assertTrue("isa Float", o instanceof Float);
             Float f = (Float)o;
             assertEquals(new Float(3.141592654), f);

             o = after.get(6);
             assertTrue("isa Double", o instanceof Double);
             Double d = (Double)o;
             assertEquals(new Double(2.99792458e8), d);

             o = after.get(7);
             assertTrue("isa Boolean", o instanceof Boolean);
             Boolean bool = (Boolean)o;
             assertTrue(bool);

             o = after.get(8);
             assertTrue("isa DataByteArray", o instanceof DataByteArray);
             DataByteArray ba = (DataByteArray)o;
             assertEquals(new DataByteArray("hello"), ba);

             o = after.get(9);
             assertTrue("isa String", o instanceof String);
             String s = (String)o;
             assertEquals("goodbye", s);
         }

         file.delete();
     }

     @Test
     public void testReadWriteInternal() throws Exception {
         // Create a tuple with every internal data type in it, and then read and
         // write it, both via DataReaderWriter and Tuple.readFields
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple(1);

         InternalMap map = new InternalMap(2);
         map.put(new Integer(1), new String("world"));
         map.put(new Long(3L), new String("all"));
         t1.set(0, map);

         File file = File.createTempFile("Tuple", "put");
         FileOutputStream fos = new FileOutputStream(file);
         DataOutput out = new DataOutputStream(fos);
         t1.write(out);
         fos.close();

         FileInputStream fis = new FileInputStream(file);
         DataInput in = new DataInputStream(fis);

         Tuple after = tf.newTuple();
         after.readFields(in);

         Object o = after.get(0);
         assertTrue("isa InternalMap", o instanceof InternalMap);

         InternalMap m = (InternalMap)o;
         assertEquals("world", (String)m.get(new Integer(1)));
         assertEquals("all", (String)m.get(new Long(3L)));
         assertNull(m.get("fred"));

         file.delete();
     }

     @Test
     public void testTupleToString() throws Exception {
         Tuple t = giveMeOneOfEach();

         assertEquals(
                 "toString",
                 "((3,3.0),{(4),(mary had a little lamb)},[hello#world,goodbye#all],42,5000000000,3.1415927,2.99792458E8,true,hello,goodbye,)",
                 t.toString());
     }

     @Test
     public void testTupleHashCode() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple(2);
         t1.set(0, new DataByteArray("hello world"));
         t1.set(1, new Integer(1));

         Tuple t2 = tf.newTuple(2);
         t2.set(0, new DataByteArray("hello world"));
         t2.set(1, new Integer(1));

         assertEquals("same data", t1.hashCode(), t2.hashCode());

         Tuple t3 = tf.newTuple(3);
         t3.set(0, new DataByteArray("hello world"));
         t3.set(1, new Integer(1));
         t3.set(2, new Long(4));
         assertFalse("different size", t1.hashCode() == t3.hashCode());

         Tuple t4 = tf.newTuple(2);
         t4.set(0, new DataByteArray("hello world"));
         t4.set(1, new Integer(2));
         assertFalse("same size, different data", t1.hashCode() == t4.hashCode());

         // Make sure we can take the hash code of all the types.
         Tuple t5 = giveMeOneOfEach();
         t5.hashCode();
     }

     @Test
     public void testTupleEquals() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple();
         Tuple t2 = tf.newTuple();

         t1.append(new Integer(3));
         t2.append(new Integer(3));

         assertFalse("different object", t1.equals(new String()));

         assertTrue("same data", t1.equals(t2));

         t2 = tf.newTuple();
         t2.append(new Integer(4));
         assertFalse("different data", t1.equals(t2));

         t2 = tf.newTuple();
         t2.append(new Integer(3));
         t2.append(new Integer(3));
         assertFalse("different size", t1.equals(t2));
     }

     @Test
     public void testTupleCompareTo() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple();
         Tuple t2 = tf.newTuple();

         t1.append(new Integer(3));
         t2.append(new Integer(3));

         assertEquals("same data equal", 0, t1.compareTo(t2));

         t2 = tf.newTuple();
         t2.append(new Integer(2));
         assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

         t2 = tf.newTuple();
         t2.append(new Integer(4));
         assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));

         t2 = tf.newTuple();
         t2.append(new Integer(3));
         t2.append(new Integer(4));
         assertTrue("less than bigger tuple", 0 > t1.compareTo(t2));

         t2 = tf.newTuple();
         assertTrue("greater than smaller tuple", 0 < t1.compareTo(t2));
     }

     @Test
     public void testMultiFieldTupleCompareTo() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple();
         Tuple t2 = tf.newTuple();

         t1.append(new DataByteArray("bbb"));
         t1.append(new DataByteArray("bbb"));
         t2.append(new DataByteArray("bbb"));
         t2.append(new DataByteArray("bbb"));

         assertEquals("same data equal", 0, t1.compareTo(t2));

         t2 = tf.newTuple();
         t2.append(new DataByteArray("aaa"));
         t2.append(new DataByteArray("aaa"));
         assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

         t2 = tf.newTuple();
         t2.append(new DataByteArray("ddd"));
         t2.append(new DataByteArray("ddd"));
         assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));

         // First column same, second lesser
         t2 = tf.newTuple();
         t2.append(new DataByteArray("bbb"));
         t2.append(new DataByteArray("aaa"));
         assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

         // First column same, second greater
         t2 = tf.newTuple();
         t2.append(new DataByteArray("bbb"));
         t2.append(new DataByteArray("ccc"));
         assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));

         // First column less, second same
         t2 = tf.newTuple();
         t2.append(new DataByteArray("aaa"));
         t2.append(new DataByteArray("bbb"));
         assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

         // First column greater, second same
         t2 = tf.newTuple();
         t2.append(new DataByteArray("ccc"));
         t2.append(new DataByteArray("bbb"));
         assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));

         // First column less, second greater
         t2 = tf.newTuple();
         t2.append(new DataByteArray("aaa"));
         t2.append(new DataByteArray("ccc"));
         assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

         // First column greater, second same
         t2 = tf.newTuple();
         t2.append(new DataByteArray("ccc"));
         t2.append(new DataByteArray("aaa"));
         assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));
     }

     @Test
     public void testByteArrayToString() throws Exception {
         DataByteArray ba = new DataByteArray("hello world");

         assertEquals("toString", "hello world", ba.toString());
     }

     @Test
     public void testByteArrayHashCode() throws Exception {
         DataByteArray ba1 = new DataByteArray("hello world");
         DataByteArray ba2 = new DataByteArray("hello world");
         DataByteArray ba3 = new DataByteArray("goodbye world");

         assertEquals("same data", ba1.hashCode(), ba2.hashCode());

         assertFalse("different data", ba1.hashCode() == ba3.hashCode());
     }

     @Test
     public void testByteArrayEquals() throws Exception {
         DataByteArray ba1 = new DataByteArray("hello world");
         DataByteArray ba2 = new DataByteArray("hello world");
         DataByteArray ba3 = new DataByteArray("goodbye world");

         assertTrue("same data", ba1.equals(ba2));

         assertFalse("different data", ba1.equals(ba3));
     }

     @Test
     public void testByteArrayCompareTo() throws Exception {
         DataByteArray ba1 = new DataByteArray("hello world");
         DataByteArray ba2 = new DataByteArray("hello world");
         DataByteArray ba3 = new DataByteArray("goodbye world");

         assertTrue("same data", ba1.compareTo(ba2) == 0);

         assertTrue("different length lexically lower value less than",
                 ba3.compareTo(ba1) < 0);
         assertTrue("different length lexically higher value greater than",
                 ba1.compareTo(ba3) > 0);

         ba2 = new DataByteArray("hello worlc");
         assertTrue("same length lexically lower value less than",
                 ba2.compareTo(ba1) < 0);
         assertTrue("same length lexically higher value greater than",
                 ba1.compareTo(ba2) > 0);

         ba2 = new DataByteArray("hello worlds");
         assertTrue("shorter lexically same value less than",
                 ba1.compareTo(ba2) < 0);
         assertTrue("longer lexically same value greater than",
                 ba2.compareTo(ba1) > 0);

     }

     @Test(expected = ExecException.class)
     public void testIntegerConversionErr() throws Exception {
         List list = new ArrayList();
         try {
             DataType.toInteger(list);
         } catch (ExecException ee) {
             assertEquals(1071, ee.getErrorCode());
             throw ee;
         }
     }

     @Test(expected = ExecException.class)
     public void testIntegerConversionErr1() throws Exception {
         DataByteArray ba = new DataByteArray("hello world");
         try {
             DataType.toInteger(ba);
         } catch (ExecException ee) {
             assertEquals(1074, ee.getErrorCode());
             throw ee;
         }
     }

     @Test(expected = ExecException.class)
     public void testTupleConversionErr() throws Exception {
         List list = new ArrayList();
         try {
             DataType.toTuple(list);
         } catch (ExecException ee) {
             assertEquals(1071, ee.getErrorCode());
             throw ee;
         }
     }

     @Test(expected = ExecException.class)
     public void testTupleConversionErr1() throws Exception {
         DataByteArray ba = new DataByteArray("hello world");
         try {
             DataType.toTuple(ba);
         } catch (ExecException ee) {
             assertEquals(1071, ee.getErrorCode());
             throw ee;
         }
     }

     @Test
     public void testByteArrayAppend() throws Exception {
         DataByteArray expected = new DataByteArray("hello world");
         DataByteArray db1 = new DataByteArray("hello ");
         DataByteArray db2 = new DataByteArray("world");
         db1.append(db2);
         assertEquals("appends as expected", db1, expected);
     }

     @Test
     public void testByteArrayAppendMore() throws Exception {
         DataByteArray expected = new DataByteArray("hello world!");
         DataByteArray db1 = new DataByteArray("hello ");
         DataByteArray db2 = new DataByteArray("world");
         DataByteArray db3 = new DataByteArray("!");
         db1.append(db2).append(db3);
         assertEquals("appends as expected", db1, expected);
     }

     @Test
     public void testByteArrayAppendBytes() throws Exception {
         DataByteArray expected = new DataByteArray("hello world");
         DataByteArray db1 = new DataByteArray("hello ");
         byte[] db2 = "world".getBytes();
         db1.append(db2);
         assertEquals("appends as expected", db1, expected);
     }

     @Test
     public void testByteArrayAppendString() throws Exception {
         DataByteArray expected = new DataByteArray("hello world");
         DataByteArray db1 = new DataByteArray("hello ");
         db1.append("world");
         assertEquals("appends as expected", db1, expected);
     }

     @Test(expected = ExecException.class)
     public void testMapConversionErr() throws Exception {
         List list = new ArrayList();
         try {
             DataType.toMap(list);
         } catch (ExecException ee) {
             assertEquals(1071, ee.getErrorCode());
             throw ee;
         }
     }

     @Test
     public void testMapConversion() throws Exception {
         Map<Integer, Float> map = new HashMap<Integer, Float>();
         DataType.toMap(map);
     }

     @Test(expected = ExecException.class)
     public void testDetermineFieldSchemaErr() throws Exception {
         List list = new ArrayList();
         try {
             DataType.determineFieldSchema(list);
             fail("Error expected.");
         } catch (ExecException ee) {
             assertEquals(1073, ee.getErrorCode());
             throw ee;
         }
     }

     private Tuple giveMeOneOfEach() throws Exception {
         TupleFactory tf = TupleFactory.getInstance();

         Tuple t1 = tf.newTuple(11);
         Tuple t2 = tf.newTuple(2);

         t2.set(0, new Integer(3));
         t2.set(1, new Float(3.0));

         DataBag bag = BagFactory.getInstance().newDefaultBag();
         bag.add(tf.newTuple(new Integer(4)));
         bag.add(tf.newTuple(new String("mary had a little lamb")));

         Map<String, Object> map = new LinkedHashMap<String, Object>(2);
         map.put(new String("hello"), new String("world"));
         map.put(new String("goodbye"), new String("all"));

         t1.set(0, t2);
         t1.set(1, bag);
         t1.set(2, map);
         t1.set(3, new Integer(42));
         t1.set(4, new Long(5000000000L));
         t1.set(5, new Float(3.141592654));
         t1.set(6, new Double(2.99792458e8));
         t1.set(7, new Boolean(true));
         t1.set(8, new DataByteArray("hello"));
         t1.set(9, new String("goodbye"));

         return t1;
     }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	* http://www.apache.org/licenses/LICENSE-2.0
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.pig.test;

	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertFalse;
	import static org.junit.Assert.assertNull;
	import static org.junit.Assert.assertTrue;
	import static org.junit.Assert.fail;

	import java.io.DataInput;
	import java.io.DataInputStream;
	import java.io.DataOutput;
	import java.io.DataOutputStream;
	import java.io.File;
	import java.io.FileInputStream;
	import java.io.FileOutputStream;
	import java.io.PipedInputStream;
	import java.io.PipedOutputStream;
	import java.util.ArrayList;
	import java.util.HashMap;
	import java.util.Iterator;
	import java.util.LinkedHashMap;
	import java.util.List;
	import java.util.Map;

	import org.apache.pig.backend.executionengine.ExecException;
	import org.apache.pig.data.BagFactory;
	import org.apache.pig.data.DataBag;
	import org.apache.pig.data.DataByteArray;
	import org.apache.pig.data.DataType;
	import org.apache.pig.data.InternalMap;
	import org.apache.pig.data.Tuple;
	import org.apache.pig.data.TupleFactory;
	import org.junit.Test;

	/**
	* This class will exercise the basic Pig data model and members. It tests for proper behavior in
	* assigment and comparision, as well as function application.
	*/
	public class TestDataModel {

	@Test
	public void testDatum() throws Exception {
	// Datum is an empty class, but in the event we
	// ever add a method, test it here
	}

	@Test
	public void testTuple() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	int arity = 5;
	int[] input1 = { 1, 2, 3, 4, 5 };
	String[] input2 = { "1", "2", "3", "4", "5" };
	String[] input3 = { "1", "2", "3", "4", "5", "6" };

	// validate construction and equality
	Tuple f1 = Util.loadFlatTuple(tf.newTuple(arity), input1);
	Tuple f2 = Util.loadFlatTuple(tf.newTuple(arity), input1);
	Tuple f3 = tf.newTuple(arity);
	assertEquals(arity, f1.size());
	assertEquals(f1, f2);

	// invalid equality
	f2 = Util.loadTuple(tf.newTuple(input3.length), input3);
	assertFalse(f1.equals(f2));

	// copy equality
	/*
	* f2.copyFrom(f1);
	* assertTrue(f1.equals(f2));
	*/

	// append function and equality
	int[] input4 = { 1, 2, 3 };
	int[] input5 = { 4, 5 };
	/*
	* f1 = Util.loadFlatTuple(tf.newTuple(input4.length), input4);
	* f2 = Util.loadFlatTuple(tf.newTuple(input5.length), input5);
	* f3 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
	* f1.appendTuple(f2);
	* assertTrue(f3.equals(f1));
	*/

	// arity then value comparision behavior
	f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1); // 1,2,3,4,5
	f2 = Util.loadFlatTuple(tf.newTuple(input4.length), input4); // 1,2,3
	assertTrue(f1.compareTo(f2) > 0);
	assertFalse(f1.compareTo(f2) < 0);

	int[] input6 = { 1, 2, 3, 4, 6 };
	f2 = Util.loadFlatTuple(tf.newTuple(input6.length), input6);
	assertTrue(f1.compareTo(f2) < 0);
	assertFalse(f1.compareTo(f2) > 0);

	// delimited export
	String expected = "1:2:3:4:5";
	f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
	assertTrue(expected.equals(f1.toDelimitedString(":")));

	// value read / write & marshalling
	PipedOutputStream pos = new PipedOutputStream();
	DataOutputStream dos = new DataOutputStream(pos);
	PipedInputStream pis = new PipedInputStream(pos);
	DataInputStream dis = new DataInputStream(pis);
	f1.write(dos);
	f1.write(dos);
	f2.readFields(dis);
	assertTrue(f1.equals(f2));
	f2.readFields(dis);
	assertTrue(f1.equals(f2));
	}

	@Test
	public void testNestTuple() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	int[][] input1 = { { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 },
	{ 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 } };
	int[][] input2 = { { 1, 2 }, { 1, 2 } };

	Tuple n1 = Util.loadNestTuple(tf.newTuple(input1.length), input1);
	Tuple n2 = tf.newTuple();

	n2 = Util.loadNestTuple(tf.newTuple(input2.length), input2);
	}

	@Test
	public void testReadWrite() throws Exception {
	// Create a tuple with every data type in it, and then read and
	// write it, both via DataReaderWriter and Tuple.readFields
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = giveMeOneOfEach();

	File file = File.createTempFile("Tuple", "put");
	FileOutputStream fos = new FileOutputStream(file);
	DataOutput out = new DataOutputStream(fos);
	t1.write(out);
	t1.write(out); // twice in a row on purpose
	fos.close();

	FileInputStream fis = new FileInputStream(file);
	DataInput in = new DataInputStream(fis);
	for (int i = 0; i < 2; i++) {
	Tuple after = tf.newTuple();
	after.readFields(in);

	Object o = after.get(0);
	assertTrue("isa Tuple", o instanceof Tuple);
	Tuple t3 = (Tuple)o;
	o = t3.get(0);
	assertTrue("isa Integer", o instanceof Integer);
	assertEquals(new Integer(3), (Integer)o);
	o = t3.get(1);
	assertTrue("isa Float", o instanceof Float);
	assertEquals(new Float(3.0), (Float)o);

	o = after.get(1);
	assertTrue("isa Bag", o instanceof DataBag);
	DataBag b = (DataBag)o;
	Iterator<Tuple> j = b.iterator();
	Tuple[] ts = new Tuple[2];
	assertTrue("first tuple in bag", j.hasNext());
	ts[0] = j.next();
	assertTrue("second tuple in bag", j.hasNext());
	ts[1] = j.next();
	o = ts[0].get(0);
	assertTrue("isa Integer", o instanceof Integer);
	assertEquals(new Integer(4), (Integer)o);
	o = ts[1].get(0);
	assertTrue("isa String", o instanceof String);
	assertEquals("mary had a little lamb", (String)o);

	o = after.get(2);
	assertTrue("isa Map", o instanceof Map);
	Map<String, Object> m = (Map<String, Object>)o;
	assertEquals("world", (String)m.get("hello"));
	assertEquals("all", (String)m.get("goodbye"));
	assertNull(m.get("fred"));

	o = after.get(3);
	assertTrue("isa Integer", o instanceof Integer);
	Integer ii = (Integer)o;
	assertEquals(new Integer(42), ii);

	o = after.get(4);
	assertTrue("isa Long", o instanceof Long);
	Long l = (Long)o;
	assertEquals(new Long(5000000000L), l);

	o = after.get(5);
	assertTrue("isa Float", o instanceof Float);
	Float f = (Float)o;
	assertEquals(new Float(3.141592654), f);

	o = after.get(6);
	assertTrue("isa Double", o instanceof Double);
	Double d = (Double)o;
	assertEquals(new Double(2.99792458e8), d);

	o = after.get(7);
	assertTrue("isa Boolean", o instanceof Boolean);
	Boolean bool = (Boolean)o;
	assertTrue(bool);

	o = after.get(8);
	assertTrue("isa DataByteArray", o instanceof DataByteArray);
	DataByteArray ba = (DataByteArray)o;
	assertEquals(new DataByteArray("hello"), ba);

	o = after.get(9);
	assertTrue("isa String", o instanceof String);
	String s = (String)o;
	assertEquals("goodbye", s);
	}

	file.delete();
	}

	@Test
	public void testReadWriteInternal() throws Exception {
	// Create a tuple with every internal data type in it, and then read and
	// write it, both via DataReaderWriter and Tuple.readFields
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple(1);

	InternalMap map = new InternalMap(2);
	map.put(new Integer(1), new String("world"));
	map.put(new Long(3L), new String("all"));
	t1.set(0, map);

	File file = File.createTempFile("Tuple", "put");
	FileOutputStream fos = new FileOutputStream(file);
	DataOutput out = new DataOutputStream(fos);
	t1.write(out);
	fos.close();

	FileInputStream fis = new FileInputStream(file);
	DataInput in = new DataInputStream(fis);

	Tuple after = tf.newTuple();
	after.readFields(in);

	Object o = after.get(0);
	assertTrue("isa InternalMap", o instanceof InternalMap);

	InternalMap m = (InternalMap)o;
	assertEquals("world", (String)m.get(new Integer(1)));
	assertEquals("all", (String)m.get(new Long(3L)));
	assertNull(m.get("fred"));

	file.delete();
	}

	@Test
	public void testTupleToString() throws Exception {
	Tuple t = giveMeOneOfEach();

	assertEquals(
	"toString",
	"((3,3.0),{(4),(mary had a little lamb)},[hello#world,goodbye#all],42,5000000000,3.1415927,2.99792458E8,true,hello,goodbye,)",
	t.toString());
	}

	@Test
	public void testTupleHashCode() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple(2);
	t1.set(0, new DataByteArray("hello world"));
	t1.set(1, new Integer(1));

	Tuple t2 = tf.newTuple(2);
	t2.set(0, new DataByteArray("hello world"));
	t2.set(1, new Integer(1));

	assertEquals("same data", t1.hashCode(), t2.hashCode());

	Tuple t3 = tf.newTuple(3);
	t3.set(0, new DataByteArray("hello world"));
	t3.set(1, new Integer(1));
	t3.set(2, new Long(4));
	assertFalse("different size", t1.hashCode() == t3.hashCode());

	Tuple t4 = tf.newTuple(2);
	t4.set(0, new DataByteArray("hello world"));
	t4.set(1, new Integer(2));
	assertFalse("same size, different data", t1.hashCode() == t4.hashCode());

	// Make sure we can take the hash code of all the types.
	Tuple t5 = giveMeOneOfEach();
	t5.hashCode();
	}

	@Test
	public void testTupleEquals() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple();
	Tuple t2 = tf.newTuple();

	t1.append(new Integer(3));
	t2.append(new Integer(3));

	assertFalse("different object", t1.equals(new String()));

	assertTrue("same data", t1.equals(t2));

	t2 = tf.newTuple();
	t2.append(new Integer(4));
	assertFalse("different data", t1.equals(t2));

	t2 = tf.newTuple();
	t2.append(new Integer(3));
	t2.append(new Integer(3));
	assertFalse("different size", t1.equals(t2));
	}

	@Test
	public void testTupleCompareTo() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple();
	Tuple t2 = tf.newTuple();

	t1.append(new Integer(3));
	t2.append(new Integer(3));

	assertEquals("same data equal", 0, t1.compareTo(t2));

	t2 = tf.newTuple();
	t2.append(new Integer(2));
	assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

	t2 = tf.newTuple();
	t2.append(new Integer(4));
	assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));

	t2 = tf.newTuple();
	t2.append(new Integer(3));
	t2.append(new Integer(4));
	assertTrue("less than bigger tuple", 0 > t1.compareTo(t2));

	t2 = tf.newTuple();
	assertTrue("greater than smaller tuple", 0 < t1.compareTo(t2));
	}

	@Test
	public void testMultiFieldTupleCompareTo() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple();
	Tuple t2 = tf.newTuple();

	t1.append(new DataByteArray("bbb"));
	t1.append(new DataByteArray("bbb"));
	t2.append(new DataByteArray("bbb"));
	t2.append(new DataByteArray("bbb"));

	assertEquals("same data equal", 0, t1.compareTo(t2));

	t2 = tf.newTuple();
	t2.append(new DataByteArray("aaa"));
	t2.append(new DataByteArray("aaa"));
	assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

	t2 = tf.newTuple();
	t2.append(new DataByteArray("ddd"));
	t2.append(new DataByteArray("ddd"));
	assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));

	// First column same, second lesser
	t2 = tf.newTuple();
	t2.append(new DataByteArray("bbb"));
	t2.append(new DataByteArray("aaa"));
	assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

	// First column same, second greater
	t2 = tf.newTuple();
	t2.append(new DataByteArray("bbb"));
	t2.append(new DataByteArray("ccc"));
	assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));

	// First column less, second same
	t2 = tf.newTuple();
	t2.append(new DataByteArray("aaa"));
	t2.append(new DataByteArray("bbb"));
	assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

	// First column greater, second same
	t2 = tf.newTuple();
	t2.append(new DataByteArray("ccc"));
	t2.append(new DataByteArray("bbb"));
	assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));

	// First column less, second greater
	t2 = tf.newTuple();
	t2.append(new DataByteArray("aaa"));
	t2.append(new DataByteArray("ccc"));
	assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));

	// First column greater, second same
	t2 = tf.newTuple();
	t2.append(new DataByteArray("ccc"));
	t2.append(new DataByteArray("aaa"));
	assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));
	}

	@Test
	public void testByteArrayToString() throws Exception {
	DataByteArray ba = new DataByteArray("hello world");

	assertEquals("toString", "hello world", ba.toString());
	}

	@Test
	public void testByteArrayHashCode() throws Exception {
	DataByteArray ba1 = new DataByteArray("hello world");
	DataByteArray ba2 = new DataByteArray("hello world");
	DataByteArray ba3 = new DataByteArray("goodbye world");

	assertEquals("same data", ba1.hashCode(), ba2.hashCode());

	assertFalse("different data", ba1.hashCode() == ba3.hashCode());
	}

	@Test
	public void testByteArrayEquals() throws Exception {
	DataByteArray ba1 = new DataByteArray("hello world");
	DataByteArray ba2 = new DataByteArray("hello world");
	DataByteArray ba3 = new DataByteArray("goodbye world");

	assertTrue("same data", ba1.equals(ba2));

	assertFalse("different data", ba1.equals(ba3));
	}

	@Test
	public void testByteArrayCompareTo() throws Exception {
	DataByteArray ba1 = new DataByteArray("hello world");
	DataByteArray ba2 = new DataByteArray("hello world");
	DataByteArray ba3 = new DataByteArray("goodbye world");

	assertTrue("same data", ba1.compareTo(ba2) == 0);

	assertTrue("different length lexically lower value less than",
	ba3.compareTo(ba1) < 0);
	assertTrue("different length lexically higher value greater than",
	ba1.compareTo(ba3) > 0);

	ba2 = new DataByteArray("hello worlc");
	assertTrue("same length lexically lower value less than",
	ba2.compareTo(ba1) < 0);
	assertTrue("same length lexically higher value greater than",
	ba1.compareTo(ba2) > 0);

	ba2 = new DataByteArray("hello worlds");
	assertTrue("shorter lexically same value less than",
	ba1.compareTo(ba2) < 0);
	assertTrue("longer lexically same value greater than",
	ba2.compareTo(ba1) > 0);

	}

	@Test(expected = ExecException.class)
	public void testIntegerConversionErr() throws Exception {
	List list = new ArrayList();
	try {
	DataType.toInteger(list);
	} catch (ExecException ee) {
	assertEquals(1071, ee.getErrorCode());
	throw ee;
	}
	}

	@Test(expected = ExecException.class)
	public void testIntegerConversionErr1() throws Exception {
	DataByteArray ba = new DataByteArray("hello world");
	try {
	DataType.toInteger(ba);
	} catch (ExecException ee) {
	assertEquals(1074, ee.getErrorCode());
	throw ee;
	}
	}

	@Test(expected = ExecException.class)
	public void testTupleConversionErr() throws Exception {
	List list = new ArrayList();
	try {
	DataType.toTuple(list);
	} catch (ExecException ee) {
	assertEquals(1071, ee.getErrorCode());
	throw ee;
	}
	}

	@Test(expected = ExecException.class)
	public void testTupleConversionErr1() throws Exception {
	DataByteArray ba = new DataByteArray("hello world");
	try {
	DataType.toTuple(ba);
	} catch (ExecException ee) {
	assertEquals(1071, ee.getErrorCode());
	throw ee;
	}
	}

	@Test
	public void testByteArrayAppend() throws Exception {
	DataByteArray expected = new DataByteArray("hello world");
	DataByteArray db1 = new DataByteArray("hello ");
	DataByteArray db2 = new DataByteArray("world");
	db1.append(db2);
	assertEquals("appends as expected", db1, expected);
	}

	@Test
	public void testByteArrayAppendMore() throws Exception {
	DataByteArray expected = new DataByteArray("hello world!");
	DataByteArray db1 = new DataByteArray("hello ");
	DataByteArray db2 = new DataByteArray("world");
	DataByteArray db3 = new DataByteArray("!");
	db1.append(db2).append(db3);
	assertEquals("appends as expected", db1, expected);
	}

	@Test
	public void testByteArrayAppendBytes() throws Exception {
	DataByteArray expected = new DataByteArray("hello world");
	DataByteArray db1 = new DataByteArray("hello ");
	byte[] db2 = "world".getBytes();
	db1.append(db2);
	assertEquals("appends as expected", db1, expected);
	}

	@Test
	public void testByteArrayAppendString() throws Exception {
	DataByteArray expected = new DataByteArray("hello world");
	DataByteArray db1 = new DataByteArray("hello ");
	db1.append("world");
	assertEquals("appends as expected", db1, expected);
	}

	@Test(expected = ExecException.class)
	public void testMapConversionErr() throws Exception {
	List list = new ArrayList();
	try {
	DataType.toMap(list);
	} catch (ExecException ee) {
	assertEquals(1071, ee.getErrorCode());
	throw ee;
	}
	}

	@Test
	public void testMapConversion() throws Exception {
	Map<Integer, Float> map = new HashMap<Integer, Float>();
	DataType.toMap(map);
	}

	@Test(expected = ExecException.class)
	public void testDetermineFieldSchemaErr() throws Exception {
	List list = new ArrayList();
	try {
	DataType.determineFieldSchema(list);
	fail("Error expected.");
	} catch (ExecException ee) {
	assertEquals(1073, ee.getErrorCode());
	throw ee;
	}
	}

	private Tuple giveMeOneOfEach() throws Exception {
	TupleFactory tf = TupleFactory.getInstance();

	Tuple t1 = tf.newTuple(11);
	Tuple t2 = tf.newTuple(2);

	t2.set(0, new Integer(3));
	t2.set(1, new Float(3.0));

	DataBag bag = BagFactory.getInstance().newDefaultBag();
	bag.add(tf.newTuple(new Integer(4)));
	bag.add(tf.newTuple(new String("mary had a little lamb")));

	Map<String, Object> map = new LinkedHashMap<String, Object>(2);
	map.put(new String("hello"), new String("world"));
	map.put(new String("goodbye"), new String("all"));

	t1.set(0, t2);
	t1.set(1, bag);
	t1.set(2, map);
	t1.set(3, new Integer(42));
	t1.set(4, new Long(5000000000L));
	t1.set(5, new Float(3.141592654));
	t1.set(6, new Double(2.99792458e8));
	t1.set(7, new Boolean(true));
	t1.set(8, new DataByteArray("hello"));
	t1.set(9, new String("goodbye"));

	return t1;
	}
	}