lens-query-lib/src/test/java/org/apache/lens/lib/query/TestFileSerdeFormatter.java - lens - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
  */
 package org.apache.lens.lib.query;

 import java.io.IOException;
 import java.util.*;

 import org.apache.lens.api.query.ResultRow;
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.query.InMemoryOutputFormatter;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.io.Text;

 import org.testng.Assert;
 import org.testng.annotations.Test;

 /**
  * The Class TestFileSerdeFormatter.
  */
 public class TestFileSerdeFormatter extends TestAbstractFileFormatter {

   /*
    * (non-Javadoc)
    *
    * @see org.apache.lens.lib.query.TestAbstractFileFormatter#testFormatter()
    */
   @Test
   public void testFormatter() throws IOException {
     super.testFormatter();
     validateSerde(LensConfConstants.DEFAULT_OUTPUT_SERDE, Text.class.getCanonicalName());
   }

   /**
    * Test serde.
    *
    * @throws IOException Signals that an I/O exception has occurred.
    */
   @Test
   public void testSerde() throws IOException {
     Configuration conf = new Configuration();
     conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
     conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
     testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
       getMockedResultSetWithoutComma());
     validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());

     // validate rows
     Assert.assertEquals(readFinalOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
       getExpectedTextRowsWithoutComma());
   }

   /**
    * Test compression with custom serde.
    *
    * @throws IOException Signals that an I/O exception has occurred.
    */
   @Test
   public void testCompressionWithCustomSerde() throws IOException {
     Configuration conf = new Configuration();
     conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
     conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
     conf.setBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION, true);
     testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt.gz",
       getMockedResultSetWithoutComma());
     validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());
     // validate rows
     Assert.assertEquals(readCompressedFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
       getExpectedTextRowsWithoutComma());
   }

   /**
    * Test text file with zip formatter.
    *
    * @throws IOException Signals that an I/O exception has occurred.
    */
   @Test
   public void testTextFileWithZipFormatter() throws IOException {
     Configuration conf = new Configuration();
     setConf(conf);
     conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
     conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
     conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
     conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
     testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
       getMockedResultSetWithoutComma());
     // validate rows
     List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
     Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
   }

   /**
    * Test csv with zip formatter.
    *
    * @throws IOException Signals that an I/O exception has occurred.
    */
   @Test
   public void testCSVWithZipFormatter() throws IOException {
     Configuration conf = new Configuration();
     setConf(conf);
     conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
     conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
     testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip", getMockedResultSet());
     // validate rows
     List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
     Assert.assertEquals(actual, getExpectedCSVRowsWithMultiple());
   }

   /**
    * Validate serde.
    *
    * @param serdeClassName      the serde class name
    * @param serializedClassName the serialized class name
    */
   private void validateSerde(String serdeClassName, String serializedClassName) {
     // check serde
     SerDe outputSerde = ((FileSerdeFormatter) formatter).getSerde();
     Assert.assertEquals(serdeClassName, outputSerde.getClass().getCanonicalName());
     Assert.assertEquals(serializedClassName, outputSerde.getSerializedClass().getCanonicalName());

   }

   private List<ResultRow> getTestRows() {
     List<ResultRow> rows = new ArrayList<ResultRow>();
     List<Object> elements = new ArrayList<Object>();
     Map<Integer, String> mapElements = new LinkedHashMap<Integer, String>();
     mapElements.put(1, "one");
     elements.add(1);
     elements.add("one");
     elements.add("one");
     elements.add("one");
     elements.add(Arrays.asList(new Byte((byte) 1)));
     elements.add(Arrays.asList(1, "one"));
     elements.add(mapElements);
     rows.add(new ResultRow(elements));

     mapElements = new LinkedHashMap<Integer, String>();
     mapElements.put(1, "one");
     mapElements.put(2, "two, 3=three");
     elements = new ArrayList<Object>();
     elements.add(2);
     elements.add("two");
     elements.add("two");
     elements.add("two");
     elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2)));
     elements.add(Arrays.asList(2, "two"));
     elements.add(mapElements);
     rows.add(new ResultRow(elements));

     mapElements = new LinkedHashMap<Integer, String>();
     mapElements.put(1, "one");
     mapElements.put(2, "two");
     mapElements.put(null, "three");
     elements = new ArrayList<Object>();
     elements.add(null);
     elements.add("three");
     elements.add("three");
     elements.add("three");
     elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null));
     elements.add(Arrays.asList(null, "three"));
     elements.add(mapElements);
     rows.add(new ResultRow(elements));

     mapElements = new LinkedHashMap<Integer, String>();
     mapElements.put(1, "one");
     mapElements.put(2, "two");
     mapElements.put(null, "three");
     mapElements.put(4, null);
     elements = new ArrayList<Object>();
     elements.add(4);
     elements.add(null);
     elements.add(null);
     elements.add(null);
     elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null, new Byte((byte) 4)));
     elements.add(Arrays.asList(4, null));
     elements.add(mapElements);
     rows.add(new ResultRow(elements));

     mapElements = new LinkedHashMap<Integer, String>();
     mapElements.put(1, "one");
     mapElements.put(2, "two");
     mapElements.put(null, "three");
     mapElements.put(4, null);
     mapElements.put(5, null);
     elements = new ArrayList<Object>();
     elements.add(null);
     elements.add(null);
     elements.add(null);
     elements.add(null);
     elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null, new Byte((byte) 4), null));
     elements.add(Arrays.asList(null, null));
     elements.add(mapElements);
     rows.add(new ResultRow(elements));

     return rows;
   }

   /*
    * (non-Javadoc)
    *
    * @see org.apache.lens.lib.query.TestAbstractFileFormatter#createFormatter()
    */
   @Override
   protected WrappedFileFormatter createFormatter() {
     return new FileSerdeFormatter();
   }

   /*
    * (non-Javadoc)
    *
    * @see org.apache.lens.lib.query.TestAbstractFileFormatter#writeAllRows(org.apache.hadoop.conf.Configuration)
    */
   @Override
   protected void writeAllRows(Configuration conf) throws IOException {
     for (ResultRow row : getTestRows()) {
       ((InMemoryOutputFormatter) formatter).writeRow(row);
     }
   }

   protected List<String> getExpectedCSVRows() {
     return new ArrayList<String>() {
       {
         add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
         add("\"1\",\"one\",\"one\",\"one\",\"[1]\",\"[1, one]\",\"{1=one}\"");
         add("\"2\",\"two\",\"two\",\"two\",\"[1, 2]\",\"[2, two]\",\"{1=one, 2=two, 3=three}\"");
         add("\"NULL\",\"three\",\"three\",\"three\",\"[1, 2, null]\",\"[null, three]\",\"{1=one, 2=two, null=three}\"");
         add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4]\",\"[4, null]\","
           + "\"{1=one, 2=two, null=three, 4=null}\"");
         add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4, null]\",\"[null, null]\","
           + "\"{1=one, 2=two, null=three, 4=null, 5=null}\"");
         add("Total rows:5");
       }
     };
   }

   protected List<String> getExpectedTextRows() {
     List<String> txtRows = new ArrayList<String>();
     txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
     txtRows.add("1oneoneone            11one1one       ");
     txtRows.add("2twotwotwo            122two1one       2two       ");
     txtRows.add("\\Nthreethreethree          12\\N\\Nthree1one       2two       \\Nthree     ");
     txtRows.add("4\\N\\N\\N12\\N44\\N1one       2two       \\Nthree     4\\N");
     txtRows.add("\\N\\N\\N\\N12\\N4\\N\\N\\N1one       2two       \\Nthree     4\\N5\\N");
     txtRows.add("Total rows:5");
     return txtRows;
   }

   protected List<String> getExpectedCSVRowsWithoutComma() {
     List<String> csvRows = new ArrayList<String>();
     csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
     csvRows.add("\"1\",\"one\",\"one\",\"one\",\"1\",\"1:one\",\"1=one\"");
     csvRows.add("\"2\",\"two\",\"two\",\"two\",\"1,2\",\"2:two\",\"1=one,2=two\"");
     csvRows.add("\"NULL\",\"three\",\"three\",\"three\",\"1,2,NULL\",\"NULL:three\",\"1=one,2=two,NULL=three\"");
     csvRows.add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4\",\"4:NULL\",\"1=one,2=two,NULL=three,4=NULL\"");
     csvRows
       .add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4,NULL\",\"NULL:NULL\","
         + "\"1=one,2=two,NULL=three,4=NULL,5=NULL\"");
     csvRows.add("Total rows:5");
     return csvRows;
   }

   protected List<String> getExpectedTextRowsWithoutComma() {
     return new ArrayList<String>() {
       {
         add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
         add("1\u0001one\u0001one\u0001one            \u0001[1]\u0001[1, one]\u0001{1=one}");
         add("2\u0001two\u0001two\u0001two            \u0001[1, 2]\u0001[2, two]\u0001{1=one, 2=two, 3=three}");
         add("\\N\u0001three\u0001three\u0001three          \u0001[1, 2, null]\u0001"
           + "[null, three]\u0001{1=one, 2=two, null=three}");
         add("4\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4]\u0001[4, null]\u0001{1=one, 2=two, null=three, 4=null}");
         add("\\N\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4, null]\u0001[null, null]"
           + "\u0001{1=one, 2=two, null=three, 4=null, 5=null}");
         add("Total rows:5");
       }
     };
   }

   protected List<String> getExpectedCSVRowsWithMultiple() {
     return new ArrayList<String>() {
       {
         add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
         add("\"1\",\"one\",\"one\",\"one\",\"[1]\",\"[1, one]\",\"{1=one}\"");
         add("\"2\",\"two\",\"two\",\"two\",\"[1, 2]\",\"[2, two]\",\"{1=one, 2=two, 3=three}\"");
         add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
         add("\"NULL\",\"three\",\"three\",\"three\",\"[1, 2, null]\",\"[null, three]\",\"{1=one, 2=two, null=three}\"");
         add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4]\",\"[4, null]\","
           + "\"{1=one, 2=two, null=three, 4=null}\"");
         add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
         add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4, null]\","
           + "\"[null, null]\",\"{1=one, 2=two, null=three, 4=null, 5=null}\"");
         add("Total rows:5");
       }
     };
   }

   protected List<String> getExpectedTextRowsWithMultiple() {
     List<String> txtRows = new ArrayList<String>();
     txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
     txtRows.add("1oneoneone            11one1one       ");
     txtRows.add("2twotwotwo            122two1one       2two       ");
     txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
     txtRows.add("\\Nthreethreethree          12\\N\\Nthree1one       2two       \\Nthree     ");
     txtRows.add("4\\N\\N\\N12\\N44\\N1one       2two       \\Nthree     4\\N");
     txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
     txtRows.add("\\N\\N\\N\\N12\\N4\\N\\N\\N1one       2two       \\Nthree     4\\N5\\N");
     txtRows.add("Total rows:5");
     return txtRows;
   }

   protected List<String> getExpectedCSVRowsWithMultipleWithoutComma() {
     List<String> csvRows = new ArrayList<String>();
     csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
     csvRows.add("\"1\",\"one\",\"one\",\"one\",\"1\",\"1:one\",\"1=one\"");
     csvRows.add("\"2\",\"two\",\"two\",\"two\",\"1,2\",\"2:two\",\"1=one,2=two\"");
     csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
     csvRows.add("\"NULL\",\"three\",\"three\",\"three\",\"1,2,NULL\",\"NULL:three\",\"1=one,2=two,NULL=three\"");
     csvRows.add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4\",\"4:NULL\",\"1=one,2=two,NULL=three,4=NULL\"");
     csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
     csvRows
       .add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4,NULL\",\"NULL:NULL\","
         + "\"1=one,2=two,NULL=three,4=NULL,5=NULL\"");
     csvRows.add("Total rows:5");
     return csvRows;
   }

   protected List<String> getExpectedTextRowsWithMultipleWithoutComma() {
     return new ArrayList<String>() {
       {
         add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
         add("1\u0001one\u0001one\u0001one            \u0001[1]\u0001[1, one]\u0001{1=one}");
         add("2\u0001two\u0001two\u0001two            \u0001[1, 2]\u0001[2, two]\u0001{1=one, 2=two, 3=three}");
         add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
         add("\\N\u0001three\u0001three\u0001three          \u0001[1, 2, null]"
           + "\u0001[null, three]\u0001{1=one, 2=two, null=three}");
         add("4\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4]\u0001[4, null]\u0001{1=one, 2=two, null=three, 4=null}");
         add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
         add("\\N\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4, null]"
           + "\u0001[null, null]\u0001{1=one, 2=two, null=three, 4=null, 5=null}");
         add("Total rows:5");
       }
     };
   }

 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing,
	* software distributed under the License is distributed on an
	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	* KIND, either express or implied. See the License for the
	* specific language governing permissions and limitations
	* under the License.
	*/
	package org.apache.lens.lib.query;

	import java.io.IOException;
	import java.util.*;

	import org.apache.lens.api.query.ResultRow;
	import org.apache.lens.server.api.LensConfConstants;
	import org.apache.lens.server.api.query.InMemoryOutputFormatter;

	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.hive.serde2.SerDe;
	import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
	import org.apache.hadoop.io.Text;

	import org.testng.Assert;
	import org.testng.annotations.Test;

	/**
	* The Class TestFileSerdeFormatter.
	*/
	public class TestFileSerdeFormatter extends TestAbstractFileFormatter {

	/*
	* (non-Javadoc)
	*
	* @see org.apache.lens.lib.query.TestAbstractFileFormatter#testFormatter()
	*/
	@Test
	public void testFormatter() throws IOException {
	super.testFormatter();
	validateSerde(LensConfConstants.DEFAULT_OUTPUT_SERDE, Text.class.getCanonicalName());
	}

	/**
	* Test serde.
	*
	* @throws IOException Signals that an I/O exception has occurred.
	*/
	@Test
	public void testSerde() throws IOException {
	Configuration conf = new Configuration();
	conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
	conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
	testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
	getMockedResultSetWithoutComma());
	validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());

	// validate rows
	Assert.assertEquals(readFinalOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
	getExpectedTextRowsWithoutComma());
	}

	/**
	* Test compression with custom serde.
	*
	* @throws IOException Signals that an I/O exception has occurred.
	*/
	@Test
	public void testCompressionWithCustomSerde() throws IOException {
	Configuration conf = new Configuration();
	conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
	conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
	conf.setBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION, true);
	testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt.gz",
	getMockedResultSetWithoutComma());
	validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());
	// validate rows
	Assert.assertEquals(readCompressedFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
	getExpectedTextRowsWithoutComma());
	}

	/**
	* Test text file with zip formatter.
	*
	* @throws IOException Signals that an I/O exception has occurred.
	*/
	@Test
	public void testTextFileWithZipFormatter() throws IOException {
	Configuration conf = new Configuration();
	setConf(conf);
	conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
	conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
	conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
	conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
	testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
	getMockedResultSetWithoutComma());
	// validate rows
	List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
	Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
	}

	/**
	* Test csv with zip formatter.
	*
	* @throws IOException Signals that an I/O exception has occurred.
	*/
	@Test
	public void testCSVWithZipFormatter() throws IOException {
	Configuration conf = new Configuration();
	setConf(conf);
	conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
	conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
	testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip", getMockedResultSet());
	// validate rows
	List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
	Assert.assertEquals(actual, getExpectedCSVRowsWithMultiple());
	}

	/**
	* Validate serde.
	*
	* @param serdeClassName the serde class name
	* @param serializedClassName the serialized class name
	*/
	private void validateSerde(String serdeClassName, String serializedClassName) {
	// check serde
	SerDe outputSerde = ((FileSerdeFormatter) formatter).getSerde();
	Assert.assertEquals(serdeClassName, outputSerde.getClass().getCanonicalName());
	Assert.assertEquals(serializedClassName, outputSerde.getSerializedClass().getCanonicalName());

	}

	private List<ResultRow> getTestRows() {
	List<ResultRow> rows = new ArrayList<ResultRow>();
	List<Object> elements = new ArrayList<Object>();
	Map<Integer, String> mapElements = new LinkedHashMap<Integer, String>();
	mapElements.put(1, "one");
	elements.add(1);
	elements.add("one");
	elements.add("one");
	elements.add("one");
	elements.add(Arrays.asList(new Byte((byte) 1)));
	elements.add(Arrays.asList(1, "one"));
	elements.add(mapElements);
	rows.add(new ResultRow(elements));

	mapElements = new LinkedHashMap<Integer, String>();
	mapElements.put(1, "one");
	mapElements.put(2, "two, 3=three");
	elements = new ArrayList<Object>();
	elements.add(2);
	elements.add("two");
	elements.add("two");
	elements.add("two");
	elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2)));
	elements.add(Arrays.asList(2, "two"));
	elements.add(mapElements);
	rows.add(new ResultRow(elements));

	mapElements = new LinkedHashMap<Integer, String>();
	mapElements.put(1, "one");
	mapElements.put(2, "two");
	mapElements.put(null, "three");
	elements = new ArrayList<Object>();
	elements.add(null);
	elements.add("three");
	elements.add("three");
	elements.add("three");
	elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null));
	elements.add(Arrays.asList(null, "three"));
	elements.add(mapElements);
	rows.add(new ResultRow(elements));

	mapElements = new LinkedHashMap<Integer, String>();
	mapElements.put(1, "one");
	mapElements.put(2, "two");
	mapElements.put(null, "three");
	mapElements.put(4, null);
	elements = new ArrayList<Object>();
	elements.add(4);
	elements.add(null);
	elements.add(null);
	elements.add(null);
	elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null, new Byte((byte) 4)));
	elements.add(Arrays.asList(4, null));
	elements.add(mapElements);
	rows.add(new ResultRow(elements));

	mapElements = new LinkedHashMap<Integer, String>();
	mapElements.put(1, "one");
	mapElements.put(2, "two");
	mapElements.put(null, "three");
	mapElements.put(4, null);
	mapElements.put(5, null);
	elements = new ArrayList<Object>();
	elements.add(null);
	elements.add(null);
	elements.add(null);
	elements.add(null);
	elements.add(Arrays.asList(new Byte((byte) 1), new Byte((byte) 2), null, new Byte((byte) 4), null));
	elements.add(Arrays.asList(null, null));
	elements.add(mapElements);
	rows.add(new ResultRow(elements));

	return rows;
	}

	/*
	* (non-Javadoc)
	*
	* @see org.apache.lens.lib.query.TestAbstractFileFormatter#createFormatter()
	*/
	@Override
	protected WrappedFileFormatter createFormatter() {
	return new FileSerdeFormatter();
	}

	/*
	* (non-Javadoc)
	*
	* @see org.apache.lens.lib.query.TestAbstractFileFormatter#writeAllRows(org.apache.hadoop.conf.Configuration)
	*/
	@Override
	protected void writeAllRows(Configuration conf) throws IOException {
	for (ResultRow row : getTestRows()) {
	((InMemoryOutputFormatter) formatter).writeRow(row);
	}
	}

	protected List<String> getExpectedCSVRows() {
	return new ArrayList<String>() {
	{
	add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	add("\"1\",\"one\",\"one\",\"one\",\"[1]\",\"[1, one]\",\"{1=one}\"");
	add("\"2\",\"two\",\"two\",\"two\",\"[1, 2]\",\"[2, two]\",\"{1=one, 2=two, 3=three}\"");
	add("\"NULL\",\"three\",\"three\",\"three\",\"[1, 2, null]\",\"[null, three]\",\"{1=one, 2=two, null=three}\"");
	add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4]\",\"[4, null]\","
	+ "\"{1=one, 2=two, null=three, 4=null}\"");
	add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4, null]\",\"[null, null]\","
	+ "\"{1=one, 2=two, null=three, 4=null, 5=null}\"");
	add("Total rows:5");
	}
	};
	}

	protected List<String> getExpectedTextRows() {
	List<String> txtRows = new ArrayList<String>();
	txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
	txtRows.add("1oneoneone 11one1one ");
	txtRows.add("2twotwotwo 122two1one 2two ");
	txtRows.add("\\Nthreethreethree 12\\N\\Nthree1one 2two \\Nthree ");
	txtRows.add("4\\N\\N\\N12\\N44\\N1one 2two \\Nthree 4\\N");
	txtRows.add("\\N\\N\\N\\N12\\N4\\N\\N\\N1one 2two \\Nthree 4\\N5\\N");
	txtRows.add("Total rows:5");
	return txtRows;
	}

	protected List<String> getExpectedCSVRowsWithoutComma() {
	List<String> csvRows = new ArrayList<String>();
	csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	csvRows.add("\"1\",\"one\",\"one\",\"one\",\"1\",\"1:one\",\"1=one\"");
	csvRows.add("\"2\",\"two\",\"two\",\"two\",\"1,2\",\"2:two\",\"1=one,2=two\"");
	csvRows.add("\"NULL\",\"three\",\"three\",\"three\",\"1,2,NULL\",\"NULL:three\",\"1=one,2=two,NULL=three\"");
	csvRows.add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4\",\"4:NULL\",\"1=one,2=two,NULL=three,4=NULL\"");
	csvRows
	.add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4,NULL\",\"NULL:NULL\","
	+ "\"1=one,2=two,NULL=three,4=NULL,5=NULL\"");
	csvRows.add("Total rows:5");
	return csvRows;
	}

	protected List<String> getExpectedTextRowsWithoutComma() {
	return new ArrayList<String>() {
	{
	add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
	add("1\u0001one\u0001one\u0001one \u0001[1]\u0001[1, one]\u0001{1=one}");
	add("2\u0001two\u0001two\u0001two \u0001[1, 2]\u0001[2, two]\u0001{1=one, 2=two, 3=three}");
	add("\\N\u0001three\u0001three\u0001three \u0001[1, 2, null]\u0001"
	+ "[null, three]\u0001{1=one, 2=two, null=three}");
	add("4\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4]\u0001[4, null]\u0001{1=one, 2=two, null=three, 4=null}");
	add("\\N\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4, null]\u0001[null, null]"
	+ "\u0001{1=one, 2=two, null=three, 4=null, 5=null}");
	add("Total rows:5");
	}
	};
	}

	protected List<String> getExpectedCSVRowsWithMultiple() {
	return new ArrayList<String>() {
	{
	add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	add("\"1\",\"one\",\"one\",\"one\",\"[1]\",\"[1, one]\",\"{1=one}\"");
	add("\"2\",\"two\",\"two\",\"two\",\"[1, 2]\",\"[2, two]\",\"{1=one, 2=two, 3=three}\"");
	add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	add("\"NULL\",\"three\",\"three\",\"three\",\"[1, 2, null]\",\"[null, three]\",\"{1=one, 2=two, null=three}\"");
	add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4]\",\"[4, null]\","
	+ "\"{1=one, 2=two, null=three, 4=null}\"");
	add("\"firstcol\",\"format(secondcol,2)\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"[1, 2, null, 4, null]\","
	+ "\"[null, null]\",\"{1=one, 2=two, null=three, 4=null, 5=null}\"");
	add("Total rows:5");
	}
	};
	}

	protected List<String> getExpectedTextRowsWithMultiple() {
	List<String> txtRows = new ArrayList<String>();
	txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
	txtRows.add("1oneoneone 11one1one ");
	txtRows.add("2twotwotwo 122two1one 2two ");
	txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
	txtRows.add("\\Nthreethreethree 12\\N\\Nthree1one 2two \\Nthree ");
	txtRows.add("4\\N\\N\\N12\\N44\\N1one 2two \\Nthree 4\\N");
	txtRows.add("firstcolformat(secondcol,2)thirdcolfourthcolfifthcolsixthcolseventhcol");
	txtRows.add("\\N\\N\\N\\N12\\N4\\N\\N\\N1one 2two \\Nthree 4\\N5\\N");
	txtRows.add("Total rows:5");
	return txtRows;
	}

	protected List<String> getExpectedCSVRowsWithMultipleWithoutComma() {
	List<String> csvRows = new ArrayList<String>();
	csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	csvRows.add("\"1\",\"one\",\"one\",\"one\",\"1\",\"1:one\",\"1=one\"");
	csvRows.add("\"2\",\"two\",\"two\",\"two\",\"1,2\",\"2:two\",\"1=one,2=two\"");
	csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	csvRows.add("\"NULL\",\"three\",\"three\",\"three\",\"1,2,NULL\",\"NULL:three\",\"1=one,2=two,NULL=three\"");
	csvRows.add("\"4\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4\",\"4:NULL\",\"1=one,2=two,NULL=three,4=NULL\"");
	csvRows.add("\"firstcol\",\"secondcol\",\"thirdcol\",\"fourthcol\",\"fifthcol\",\"sixthcol\",\"seventhcol\"");
	csvRows
	.add("\"NULL\",\"NULL\",\"NULL\",\"NULL\",\"1,2,NULL,4,NULL\",\"NULL:NULL\","
	+ "\"1=one,2=two,NULL=three,4=NULL,5=NULL\"");
	csvRows.add("Total rows:5");
	return csvRows;
	}

	protected List<String> getExpectedTextRowsWithMultipleWithoutComma() {
	return new ArrayList<String>() {
	{
	add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
	add("1\u0001one\u0001one\u0001one \u0001[1]\u0001[1, one]\u0001{1=one}");
	add("2\u0001two\u0001two\u0001two \u0001[1, 2]\u0001[2, two]\u0001{1=one, 2=two, 3=three}");
	add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
	add("\\N\u0001three\u0001three\u0001three \u0001[1, 2, null]"
	+ "\u0001[null, three]\u0001{1=one, 2=two, null=three}");
	add("4\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4]\u0001[4, null]\u0001{1=one, 2=two, null=three, 4=null}");
	add("firstcol\u0001secondcol\u0001thirdcol\u0001fourthcol\u0001fifthcol\u0001sixthcol\u0001seventhcol");
	add("\\N\u0001\\N\u0001\\N\u0001\\N\u0001[1, 2, null, 4, null]"
	+ "\u0001[null, null]\u0001{1=one, 2=two, null=three, 4=null, 5=null}");
	add("Total rows:5");
	}
	};
	}

	}