HIVE-27541: Backport of HIVE-21924, HIVE-22769, HIVE-24224, HIVE-24381, HIVE-21905 to branch-3 (#4524)
* HIVE-21924: Split text files even if header/footer exists (Mustafa Iman, reviewed by Sankar Hariappan)
* HIVE-22769: Incorrect query results and query failure during split generation for compressed text files (Panos G via Ashutosh Chauhan , Mustafa Iman)
* HIVE-24224: Fix skipping header/footer for Hive on Tez on compressed file ( Panos G via Ashutosh Chauhan)s
* HIVE-24381: Compressed text input returns 0 rows if skip header/footer is mentioned (Naresh Panchetty Ramanaiah, reviewed by Panagiotis Garefalakis, Jesus Camacho Rodriguez)
* HIVE-21905: Generics improvement around the FetchOperator class (Ivan Suller via Jesus Camacho Rodriguez)
---------
Co-authored-by: Mustafa Iman <mustafa@cloudera.com>
Co-authored-by: Panos Garefalakis <pgaref@cloudera.com>
Co-authored-by: Naresh P R <prnaresh.naresh@gmail.com>
Co-authored-by: Ivan Suller <isuller@cloudera.com>
Co-authored-by: Vineet Garg <vgarg@apache.org>
Co-authored-by: Mahesh Kumar Behera <mahesh@apache.org>
Co-authored-by: Jason Dere <jdere@hortonworks.com>
---------
Signed-off-by: Sankar Hariappan <sankarh@apache.org>
Closes (#4524)
diff --git a/data/files/compressed_4line_file1.csv b/data/files/compressed_4line_file1.csv
new file mode 100644
index 0000000..efe52db
--- /dev/null
+++ b/data/files/compressed_4line_file1.csv
@@ -0,0 +1,3 @@
+1,2019-12-31
+2,2019-12-31
+3,2019-12-31
diff --git a/data/files/compressed_4line_file1.csv.bz2 b/data/files/compressed_4line_file1.csv.bz2
new file mode 100644
index 0000000..ada697d
--- /dev/null
+++ b/data/files/compressed_4line_file1.csv.bz2
Binary files differ
diff --git a/data/files/compressed_4line_file2.csv b/data/files/compressed_4line_file2.csv
new file mode 100644
index 0000000..629a850
--- /dev/null
+++ b/data/files/compressed_4line_file2.csv
@@ -0,0 +1,3 @@
+1,2019-12-31 00
+2,2019-12-31 01
+3,2019-12-31 02
diff --git a/data/files/compressed_4line_file2.csv.bz2 b/data/files/compressed_4line_file2.csv.bz2
new file mode 100644
index 0000000..4b5353a
--- /dev/null
+++ b/data/files/compressed_4line_file2.csv.bz2
Binary files differ
diff --git a/data/files/header_footer_table_4/0001.txt b/data/files/header_footer_table_4/0001.txt
new file mode 100644
index 0000000..878e3cc
--- /dev/null
+++ b/data/files/header_footer_table_4/0001.txt
@@ -0,0 +1,7 @@
+header_int,header_name,header_choice
+12,alex,daily
+3,barry,yearly
+5,chelsea,monthly
+8,xavier,monthly
+footer_int,footer_name,footer_choice
+footer2_int,footer2_name,footer2_choice
\ No newline at end of file
diff --git a/data/files/header_footer_table_4/0002.txt b/data/files/header_footer_table_4/0002.txt
new file mode 100644
index 0000000..dd27f0e
--- /dev/null
+++ b/data/files/header_footer_table_4/0002.txt
@@ -0,0 +1,7 @@
+header_int,header_name,header_choice
+9,derek,yearly
+11,ethan,monthly
+1,faith,yearly
+21,yves,daily
+footer_int,footer_name,footer_choice
+footer2_int,footer2_name,footer2_choice
\ No newline at end of file
diff --git a/data/files/test.csv.gz b/data/files/test.csv.gz
new file mode 100644
index 0000000..defb4cb
--- /dev/null
+++ b/data/files/test.csv.gz
Binary files differ
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 52cde10..dd24b42 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -42,7 +42,9 @@
# NOTE: Add tests to minitez only if it is very
# specific to tez and cannot be added to minillap.
-minitez.query.files.shared=delete_orig_table.q,\
+minitez.query.files.shared=\
+ compressed_skip_header_footer_aggr.q,\
+ delete_orig_table.q,\
orc_merge12.q,\
orc_vectorization_ppd.q,\
topnkey.q,\
@@ -397,6 +399,13 @@
minillap.query.files=acid_bucket_pruning.q,\
bucket5.q,\
bucket6.q,\
+ create_genericudaf.q,\
+ create_udaf.q,\
+ create_view.q,\
+ cte_2.q,\
+ cte_4.q,\
+ cttl.q,\
+ dynamic_partition_pruning_2.q,\
dynamic_semijoin_user_level.q,\
except_distinct.q,\
explainuser_2.q,\
@@ -410,6 +419,9 @@
reduce_deduplicate_distinct.q, \
remote_script.q,\
file_with_header_footer.q,\
+ skip_header_footer_aggr.q,\
+ skip_header_footer_proj.q,\
+ compressed_skip_header_footer_aggr.q,\
external_table_purge.q,\
external_table_with_space_in_location_path.q,\
import_exported_table.q,\
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/LineRrOffsetReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/LineRrOffsetReader.java
index 3fc1fa2..a0f949b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/LineRrOffsetReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/LineRrOffsetReader.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader.ReaderWithOffsets;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.LineRecordReader;
final class LineRrOffsetReader extends PassThruOffsetReader {
@@ -43,24 +44,29 @@ final class LineRrOffsetReader extends PassThruOffsetReader {
isCompressedMethod = isCompressedMethodTmp;
}
- static ReaderWithOffsets create(LineRecordReader sourceReader) {
- if (isCompressedMethod == null) return new PassThruOffsetReader(sourceReader);
+ static ReaderWithOffsets create(LineRecordReader sourceReader, JobConf jobConf, int skipHeaderCnt, int skipFooterCnt) {
+ // File not compressed, skipping is already done as part of SkippingTextInputFormat
+ if (isCompressedMethod == null) {
+ return new PassThruOffsetReader(sourceReader, jobConf, 0, 0);
+ }
Boolean isCompressed = null;
try {
isCompressed = (Boolean)isCompressedMethod.invoke(sourceReader);
} catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
LlapIoImpl.LOG.error("Cannot check the reader for compression; offsets not supported", e);
- return new PassThruOffsetReader(sourceReader);
+ return new PassThruOffsetReader(sourceReader, jobConf, 0, 0);
}
if (isCompressed) {
+ // Cannot slice compressed files - do header/footer skipping within the Reader
LlapIoImpl.LOG.info("Reader is compressed; offsets not supported");
- return new PassThruOffsetReader(sourceReader); // Cannot slice compressed files.
+ return new PassThruOffsetReader(sourceReader, jobConf, skipHeaderCnt, skipFooterCnt);
}
- return new LineRrOffsetReader(sourceReader);
+ // For non-compressed Text Files Header/Footer Skipping is already done as part of SkippingTextInputFormat
+ return new LineRrOffsetReader(sourceReader, jobConf);
}
- private LineRrOffsetReader(LineRecordReader sourceReader) {
- super(sourceReader);
+ private LineRrOffsetReader(LineRecordReader sourceReader, JobConf jobConf) {
+ super(sourceReader, jobConf, 0, 0);
this.lrReader = sourceReader;
this.posKey = (LongWritable)key;
}
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/PassThruOffsetReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/PassThruOffsetReader.java
index ba2b52d..56faace 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/PassThruOffsetReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/PassThruOffsetReader.java
@@ -20,23 +20,79 @@
import java.io.IOException;
import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader.ReaderWithOffsets;
+import org.apache.hadoop.hive.ql.exec.FooterBuffer;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
-@SuppressWarnings("rawtypes") class PassThruOffsetReader implements ReaderWithOffsets {
+class PassThruOffsetReader implements ReaderWithOffsets {
protected final RecordReader sourceReader;
protected final Object key;
protected final Writable value;
+ protected final JobConf jobConf;
+ protected final int skipHeaderCnt;
+ protected final int skipFooterCnt;
+ private transient FooterBuffer footerBuffer;
+ private transient boolean initialized = false;
- PassThruOffsetReader(RecordReader sourceReader) {
+ PassThruOffsetReader(RecordReader sourceReader, JobConf jobConf, int headerCnt, int footerCnt) {
this.sourceReader = sourceReader;
- key = sourceReader.createKey();
- value = (Writable)sourceReader.createValue();
+ this.key = sourceReader.createKey();
+ this.value = (Writable)sourceReader.createValue();
+ this.jobConf = jobConf;
+ this.skipHeaderCnt = headerCnt;
+ this.skipFooterCnt = footerCnt;
}
@Override
public boolean next() throws IOException {
- return sourceReader.next(key, value);
+ try {
+ boolean opNotEOF = true;
+ /**
+ * Start reading a new file.
+ * If file contains header, skip header lines before reading the records.
+ * If file contains footer, used FooterBuffer to cache and remove footer
+ * records at the end of the file.
+ */
+ if (!initialized) {
+ // Skip header lines.
+ opNotEOF = Utilities.skipHeader(sourceReader, skipHeaderCnt, key, value);
+
+ // Initialize footer buffer.
+ if (opNotEOF && skipFooterCnt > 0) {
+ footerBuffer = new FooterBuffer();
+ opNotEOF = footerBuffer.initializeBuffer(jobConf, sourceReader, skipFooterCnt, (WritableComparable) key, value);
+ }
+ this.initialized = true;
+ }
+
+ if (opNotEOF && footerBuffer == null) {
+ /**
+ * When file doesn't end after skipping header line
+ * and there is NO footer lines, read normally.
+ */
+ opNotEOF = sourceReader.next(key, value);
+ }
+
+ if (opNotEOF && footerBuffer != null) {
+ /**
+ * When file doesn't end after skipping header line
+ * and there IS footer lines, update footerBuffer
+ */
+ opNotEOF = footerBuffer.updateBuffer(jobConf, sourceReader, (WritableComparable) key, value);
+ }
+
+ if (opNotEOF) {
+ // File reached the end
+ return true;
+ }
+ // Done reading
+ return false;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
}
@Override
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
index 5b54af5..7f1dac9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
@@ -58,9 +58,11 @@
import org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer.SerDeStripeMetadata;
import org.apache.hadoop.hive.llap.io.decode.OrcEncodedDataConsumer;
import org.apache.hadoop.hive.llap.io.encoded.VectorDeserializeOrcWriter.AsyncCallback;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.HdfsUtils;
+import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.WriterOptions;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
@@ -69,6 +71,7 @@
import org.apache.hadoop.hive.ql.io.orc.encoded.CacheChunk;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -1392,8 +1395,10 @@ public void startReadSplitFromFile(
ReaderWithOffsets offsetReader = null;
@SuppressWarnings("rawtypes")
RecordReader sourceReader = sourceInputFormat.getRecordReader(split, jobConf, reporter);
+ Path path = split.getPath().getFileSystem(daemonConf).makeQualified(split.getPath());
+ PartitionDesc partDesc = HiveFileFormatUtils.getFromPathRecursively(parts, path, null);
try {
- offsetReader = createOffsetReader(sourceReader);
+ offsetReader = createOffsetReader(sourceReader, partDesc.getTableDesc());
sourceReader = null;
} finally {
if (sourceReader != null) {
@@ -1601,16 +1606,20 @@ private ObjectInspector getOiFromSerDe() throws IOException {
}
}
- private ReaderWithOffsets createOffsetReader(RecordReader<?, ?> sourceReader) {
+ private ReaderWithOffsets createOffsetReader(RecordReader<?, ?> sourceReader, TableDesc tableDesc)
+ throws IOException {
+ int headerCount = Utilities.getHeaderCount(tableDesc);
+ int footerCount = Utilities.getFooterCount(tableDesc, jobConf);
if (LlapIoImpl.LOG.isDebugEnabled()) {
- LlapIoImpl.LOG.debug("Using " + sourceReader.getClass().getSimpleName() + " to read data");
+ LlapIoImpl.LOG.debug("Using {} to read data with skip.header.line.count {} and skip.footer.line.count {}",
+ sourceReader.getClass().getSimpleName(), headerCount, footerCount);
}
// Handle the special cases here. Perhaps we could have a more general structure, or even
// a configurable set (like storage handlers), but for now we only have one.
if (isLrrEnabled && sourceReader instanceof LineRecordReader) {
- return LineRrOffsetReader.create((LineRecordReader)sourceReader);
+ return LineRrOffsetReader.create((LineRecordReader)sourceReader, jobConf, headerCount, footerCount);
}
- return new PassThruOffsetReader(sourceReader);
+ return new PassThruOffsetReader(sourceReader, jobConf, headerCount, footerCount);
}
private static String[] extractHosts(FileSplit split, boolean isInMemory) throws IOException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index 223e52b..e92432f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -213,7 +213,6 @@ public void setWork(FetchWork work) {
*/
private static final Map<String, InputFormat> inputFormats = new HashMap<String, InputFormat>();
- @SuppressWarnings("unchecked")
public static InputFormat getInputFormatFromCache(
Class<? extends InputFormat> inputFormatClass, Configuration conf) throws IOException {
if (Configurable.class.isAssignableFrom(inputFormatClass) ||
@@ -311,7 +310,7 @@ static void setFetchOperatorContext(JobConf conf, List<Path> paths) {
private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
if (!iterSplits.hasNext()) {
- FetchInputFormatSplit[] splits = getNextSplits();
+ List<FetchInputFormatSplit> splits = getNextSplits();
if (splits == null) {
return null;
}
@@ -327,7 +326,7 @@ private RecordReader<WritableComparable, Writable> getRecordReader() throws Exce
if (isPartitioned) {
row[1] = createPartValue(currDesc, partKeyOI);
}
- iterSplits = Arrays.asList(splits).iterator();
+ iterSplits = splits.iterator();
if (LOG.isDebugEnabled()) {
LOG.debug("Creating fetchTask with deserializer typeinfo: "
@@ -340,7 +339,6 @@ private RecordReader<WritableComparable, Writable> getRecordReader() throws Exce
final FetchInputFormatSplit target = iterSplits.next();
- @SuppressWarnings("unchecked")
final RecordReader<WritableComparable, Writable> reader = target.getRecordReader(job);
if (hasVC || work.getSplitSample() != null) {
currRecReader = new HiveRecordReader<WritableComparable, Writable>(reader, job) {
@@ -366,7 +364,7 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException
return currRecReader;
}
- protected FetchInputFormatSplit[] getNextSplits() throws Exception {
+ private List<FetchInputFormatSplit> getNextSplits() throws Exception {
while (getNextPath()) {
// not using FileInputFormat.setInputPaths() here because it forces a connection to the
// default file system - which may or may not be online during pure metadata operations
@@ -414,7 +412,7 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException
if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) {
Collections.sort(inputSplits, new FetchInputFormatSplitComparator());
}
- return inputSplits.toArray(new FetchInputFormatSplit[inputSplits.size()]);
+ return inputSplits;
}
return null;
@@ -640,7 +638,6 @@ public void closeOperator() throws HiveException {
*/
public void setupContext(List<Path> paths) {
this.iterPath = paths.iterator();
- List<PartitionDesc> partitionDescs;
if (!isPartitioned) {
this.iterPartDesc = Iterators.cycle(new PartitionDesc(work.getTblDesc(), null));
} else {
@@ -666,7 +663,6 @@ private StructObjectInspector setupOutputObjectInspector() throws HiveException
}
partKeyOI = getPartitionKeyOI(tableDesc);
- PartitionDesc partDesc = new PartitionDesc(tableDesc, null);
List<PartitionDesc> listParts = work.getPartDesc();
// Chose the table descriptor if none of the partitions is present.
// For eg: consider the query:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java
index be88dad0..8ead797 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java
@@ -29,7 +29,7 @@
import org.apache.hadoop.util.ReflectionUtils;
public class FooterBuffer {
- private ArrayList<ObjectPair> buffer;
+ private ArrayList<ObjectPair<WritableComparable, Writable>> buffer;
private int cur;
public FooterBuffer() {
@@ -64,13 +64,13 @@ public boolean initializeBuffer(JobConf job, RecordReader recordreader,
int footerCount, WritableComparable key, Writable value) throws IOException {
// Fill the buffer with key value pairs.
- this.buffer = new ArrayList<ObjectPair>();
+ this.buffer = new ArrayList<>();
while (buffer.size() < footerCount) {
boolean notEOF = recordreader.next(key, value);
if (!notEOF) {
return false;
}
- ObjectPair tem = new ObjectPair();
+ ObjectPair<WritableComparable, Writable> tem = new ObjectPair<>();
tem.setFirst(ReflectionUtils.copy(job, key, tem.getFirst()));
tem.setSecond(ReflectionUtils.copy(job, value, tem.getSecond()));
buffer.add(tem);
@@ -98,8 +98,8 @@ public boolean initializeBuffer(JobConf job, RecordReader recordreader,
*/
public boolean updateBuffer(JobConf job, RecordReader recordreader,
WritableComparable key, Writable value) throws IOException {
- key = ReflectionUtils.copy(job, (WritableComparable)buffer.get(cur).getFirst(), key);
- value = ReflectionUtils.copy(job, (Writable)buffer.get(cur).getSecond(), value);
+ key = ReflectionUtils.copy(job, buffer.get(cur).getFirst(), key);
+ value = ReflectionUtils.copy(job, buffer.get(cur).getSecond(), value);
boolean notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond());
if (notEOF) {
cur = (++cur) % buffer.size();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index abac436..50f5bcc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -189,7 +189,6 @@
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -3765,8 +3764,8 @@ public static File createTempDir(String baseDir){
* @return Return true if there are 0 or more records left in the file
* after skipping all headers, otherwise return false.
*/
- public static boolean skipHeader(RecordReader<WritableComparable, Writable> currRecReader,
- int headerCount, WritableComparable key, Writable value) throws IOException {
+ public static <K, V> boolean skipHeader(RecordReader<K, V> currRecReader, int headerCount, K key, V value)
+ throws IOException {
while (headerCount > 0) {
if (!currRecReader.next(key, value)) {
return false;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
index a0148f7..17f1ca9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
@@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -54,7 +55,8 @@
* a binary search to find the block to begin reading from, and stop reading once it can be
* determined no other entries will match the filter.
*/
-public abstract class HiveContextAwareRecordReader<K, V> implements RecordReader<K, V> {
+public abstract class HiveContextAwareRecordReader<K extends WritableComparable, V extends Writable>
+ implements RecordReader<K, V> {
private static final Logger LOG = LoggerFactory.getLogger(HiveContextAwareRecordReader.class.getName());
@@ -68,7 +70,7 @@ public abstract class HiveContextAwareRecordReader<K, V> implements RecordReader
private final List<Comparison> stopComparisons = new ArrayList<Comparison>();
private Map<Path, PartitionDesc> pathToPartitionInfo;
- protected RecordReader recordReader;
+ protected RecordReader<K, V> recordReader;
protected JobConf jobConf;
protected boolean isSorted = false;
@@ -76,17 +78,17 @@ public HiveContextAwareRecordReader(JobConf conf) throws IOException {
this(null, conf);
}
- public HiveContextAwareRecordReader(RecordReader recordReader) {
+ public HiveContextAwareRecordReader(RecordReader<K, V> recordReader) {
this.recordReader = recordReader;
}
- public HiveContextAwareRecordReader(RecordReader recordReader, JobConf conf)
+ public HiveContextAwareRecordReader(RecordReader<K, V> recordReader, JobConf conf)
throws IOException {
this.recordReader = recordReader;
this.jobConf = conf;
}
- public void setRecordReader(RecordReader recordReader) {
+ public void setRecordReader(RecordReader<K, V> recordReader) {
this.recordReader = recordReader;
}
@@ -339,18 +341,20 @@ public boolean doNext(K key, V value) throws IOException {
part = null;
}
TableDesc table = (part == null) ? null : part.getTableDesc();
- if (table != null) {
+ // In TextFormat, skipping is already taken care of as part of SkippingTextInputFormat.
+ // This code will be also called from LLAP when pipeline is non-vectorized and cannot create wrapper.
+ if (table != null && !TextInputFormat.class.isAssignableFrom(part.getInputFileFormatClass())) {
headerCount = Utilities.getHeaderCount(table);
footerCount = Utilities.getFooterCount(table, jobConf);
}
// If input contains header, skip header.
- if (!Utilities.skipHeader(recordReader, headerCount, (WritableComparable)key, (Writable)value)) {
+ if (!Utilities.skipHeader(recordReader, headerCount, key, value)) {
return false;
}
if (footerCount > 0) {
footerBuffer = new FooterBuffer();
- if (!footerBuffer.initializeBuffer(jobConf, recordReader, footerCount, (WritableComparable)key, (Writable)value)) {
+ if (!footerBuffer.initializeBuffer(jobConf, recordReader, footerCount, key, value)) {
return false;
}
}
@@ -360,7 +364,7 @@ public boolean doNext(K key, V value) throws IOException {
// Table files don't have footer rows.
return recordReader.next(key, value);
} else {
- return footerBuffer.updateBuffer(jobConf, recordReader, (WritableComparable)key, (Writable)value);
+ return footerBuffer.updateBuffer(jobConf, recordReader, key, value);
}
} catch (Exception e) {
return HiveIOExceptionHandlerUtil.handleRecordReaderNextException(e, jobConf);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index f617c3c..899771b 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -18,40 +18,22 @@
package org.apache.hadoop.hive.ql.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.StringInternUtils;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
-import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.hive.common.util.Ref;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.StringInternUtils;
+import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
import org.apache.hadoop.hive.llap.io.api.LlapIo;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner;
@@ -69,6 +51,7 @@
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputFormat;
@@ -77,8 +60,27 @@
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.HiveStringUtils;
+import org.apache.hive.common.util.Ref;
import org.apache.hive.common.util.ReflectionUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* HiveInputFormat is a parameterized InputFormat which looks at the path name
@@ -370,7 +372,6 @@ public static InputFormat<WritableComparable, Writable> getInputFormatFromCache(
public RecordReader getRecordReader(InputSplit split, JobConf job,
Reporter reporter) throws IOException {
HiveInputSplit hsplit = (HiveInputSplit) split;
- InputSplit inputSplit = hsplit.getInputSplit();
String inputFormatClassName = null;
Class inputFormatClass = null;
try {
@@ -411,7 +412,8 @@ public RecordReader getRecordReader(InputSplit split, JobConf job,
}
RecordReader innerReader = null;
try {
- innerReader = inputFormat.getRecordReader(inputSplit, job, reporter);
+ // Handle the special header/footer skipping cases here.
+ innerReader = RecordReaderWrapper.create(inputFormat, hsplit, part.getTableDesc(), job, reporter);
} catch (Exception e) {
innerReader = HiveIOExceptionHandlerUtil
.handleRecordReaderCreationException(e, job);
@@ -502,14 +504,19 @@ private void addSplitsForGroup(List<Path> dirs, TableScanOperator tableScan, Job
}
conf.setInputFormat(inputFormat.getClass());
- int headerCount = 0;
- int footerCount = 0;
if (table != null) {
- headerCount = Utilities.getHeaderCount(table);
- footerCount = Utilities.getFooterCount(table, conf);
+ int headerCount = Utilities.getHeaderCount(table);
+ int footerCount = Utilities.getFooterCount(table, conf);
if (headerCount != 0 || footerCount != 0) {
- // Input file has header or footer, cannot be splitted.
- HiveConf.setLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, Long.MAX_VALUE);
+ if (TextInputFormat.class.isAssignableFrom(inputFormatClass) && isUncompressedInput(finalDirs, conf)) {
+ SkippingTextInputFormat skippingTextInputFormat = new SkippingTextInputFormat();
+ skippingTextInputFormat.configure(conf, headerCount, footerCount);
+ inputFormat = skippingTextInputFormat;
+ } else {
+ // if the input is Compressed OR not text we have no way of splitting them!
+ // In that case RecordReader should take care of header/footer skipping!
+ HiveConf.setLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, Long.MAX_VALUE);
+ }
}
}
@@ -568,6 +575,25 @@ protected ValidWriteIdList getMmValidWriteIds(
return validWriteIdList;
}
+ public boolean isUncompressedInput(List<Path> finalPaths, Configuration conf) throws IOException {
+ CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(conf);
+ for (Path curr : finalPaths) {
+ FileSystem fs = curr.getFileSystem(conf);
+ if (fs.isDirectory(curr)) {
+ List<FileStatus> results = new ArrayList<>();
+ FileUtils.listStatusRecursively(fs, fs.getFileStatus(curr), results);
+ for (FileStatus fileStatus : results) {
+ if (compressionCodecs.getCodec(fileStatus.getPath()) != null) {
+ return false;
+ }
+ }
+ } else if (compressionCodecs.getCodec(curr) != null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
public static void processPathsForMmRead(List<Path> dirs, Configuration conf,
ValidWriteIdList validWriteIdList, List<Path> finalPaths,
List<Path> pathsWithFileOriginals) throws IOException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
index 3864060..359184a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java
@@ -35,12 +35,12 @@ public class HiveRecordReader<K extends WritableComparable, V extends Writable>
- public HiveRecordReader(RecordReader recordReader)
+ public HiveRecordReader(RecordReader<K, V> recordReader)
throws IOException {
super(recordReader);
}
- public HiveRecordReader(RecordReader recordReader, JobConf conf)
+ public HiveRecordReader(RecordReader<K, V> recordReader, JobConf conf)
throws IOException {
super(recordReader, conf);
}
@@ -50,14 +50,17 @@ public void doClose() throws IOException {
recordReader.close();
}
+ @Override
public K createKey() {
- return (K) recordReader.createKey();
+ return recordReader.createKey();
}
+ @Override
public V createValue() {
- return (V) recordReader.createValue();
+ return recordReader.createValue();
}
+ @Override
public long getPos() throws IOException {
return recordReader.getPos();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RecordReaderWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordReaderWrapper.java
new file mode 100644
index 0000000..811c314
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RecordReaderWrapper.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.LineRecordReader;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+class RecordReaderWrapper extends LineRecordReader {
+ private static final Method isCompressedMethod;
+ private transient boolean initialized = false;
+ protected final JobConf jobConf;
+ protected final int skipHeaderCnt;
+ protected final int skipFooterCnt;
+
+ private List<Pair<WritableComparable, Writable>> footerBuffer;
+ private int cur;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RecordReaderWrapper.class.getName());
+
+ static {
+ Method isCompressedMethodTmp;
+ try {
+ isCompressedMethodTmp = LineRecordReader.class.getDeclaredMethod("isCompressedInput");
+ isCompressedMethodTmp.setAccessible(true);
+ } catch (Throwable t) {
+ isCompressedMethodTmp = null;
+ LOG.warn("Cannot get LineRecordReader isCompressedInput method", t);
+ }
+ isCompressedMethod = isCompressedMethodTmp;
+ }
+
+ static RecordReader create(InputFormat inputFormat, HiveInputFormat.HiveInputSplit split, TableDesc tableDesc,
+ JobConf jobConf, Reporter reporter) throws IOException {
+ int headerCount = Utilities.getHeaderCount(tableDesc);
+ int footerCount = Utilities.getFooterCount(tableDesc, jobConf);
+ RecordReader innerReader = inputFormat.getRecordReader(split.getInputSplit(), jobConf, reporter);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using {} to read data with skip.header.line.count {} and skip.footer.line.count {}",
+ innerReader.getClass().getSimpleName(), headerCount, footerCount);
+ }
+
+ // For non-compressed Text Files Header/Footer Skipping is already done as part of SkippingTextInputFormat
+ if (innerReader instanceof LineRecordReader) {
+ // File not compressed, skipping is already done as part of SkippingTextInputFormat
+ if (isCompressedMethod == null) {
+ return innerReader;
+ }
+ Boolean isCompressed = null;
+ try {
+ isCompressed = (Boolean) isCompressedMethod.invoke(innerReader);
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+ LOG.error("Cannot check the reader for compression; offsets not supported", e);
+ return innerReader;
+ }
+ if (isCompressed && (headerCount > 0 || footerCount > 0)) {
+ // Cannot slice compressed files - do header/footer skipping within the Reader
+ LOG.info("Reader is compressed; offsets not supported");
+ return new RecordReaderWrapper(split, jobConf, headerCount, footerCount);
+ }
+ }
+ return innerReader;
+ }
+
+ private RecordReaderWrapper(FileSplit split, JobConf jobConf, int headerCnt, int footerCnt) throws IOException {
+ super(jobConf, split);
+ this.jobConf = jobConf;
+ this.skipHeaderCnt = headerCnt;
+ this.skipFooterCnt = footerCnt;
+ }
+
+ @Override
+ public synchronized boolean next(LongWritable key, Text value) throws IOException {
+ try {
+ boolean opNotEOF = true;
+ /**
+ * Start reading a new file.
+ * If file contains header, skip header lines before reading the records.
+ * If file contains footer, used FooterBuffer to cache and remove footer
+ * records at the end of the file.
+ */
+ if (!initialized) {
+ // Skip header lines.
+ opNotEOF = skipHeader(skipHeaderCnt, key, value);
+
+ // Initialize footer buffer.
+ if (opNotEOF && skipFooterCnt > 0) {
+ opNotEOF = initializeBuffer(jobConf, skipFooterCnt, key, value);
+ }
+ this.initialized = true;
+ }
+
+ if (opNotEOF && footerBuffer == null) {
+ /**
+ * When file doesn't end after skipping header line
+ * and there is NO footer lines, read normally.
+ */
+ opNotEOF = super.next(key, value);
+ }
+
+ if (opNotEOF && footerBuffer != null) {
+ /**
+ * When file doesn't end after skipping header line
+ * and there IS footer lines, update footerBuffer
+ */
+ opNotEOF = updateBuffer(jobConf, key, value);
+ }
+
+ if (opNotEOF) {
+ // File reached the end
+ return true;
+ }
+ // Done reading
+ return false;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private boolean skipHeader(int headerCount, LongWritable key, Text value) throws IOException {
+ while (headerCount > 0) {
+ if (!super.next(key, value)) {
+ return false;
+ }
+ headerCount--;
+ }
+ return true;
+ }
+
+ public boolean initializeBuffer(JobConf job, int footerCount, LongWritable key, Text value) throws IOException {
+ // Fill the buffer with key value pairs.
+ this.footerBuffer = new ArrayList<>();
+ while (footerBuffer.size() < footerCount) {
+ boolean notEOF = super.next(key, value);
+ if (!notEOF) {
+ return false;
+ }
+ WritableComparable left = ReflectionUtils.copy(job, key, null);
+ Writable right = ReflectionUtils.copy(job, value, null);
+ Pair<WritableComparable, Writable> tem = Pair.of(left, right);
+ footerBuffer.add(tem);
+ }
+ this.cur = 0;
+ return true;
+ }
+
+ /**
+ * Enqueue most recent record read, and dequeue earliest result in the queue.
+ *
+ * @param job
+ * Current job configuration.
+ * @param key
+ * Key of current reading record.
+ *
+ * @param value
+ * Value of current reading record.
+ *
+ * @return Return false if reaches the end of file, otherwise return true.
+ */
+ public boolean updateBuffer(JobConf job, WritableComparable key, Writable value) throws IOException {
+ key = ReflectionUtils.copy(job, footerBuffer.get(cur).getKey(), key);
+ value = ReflectionUtils.copy(job, footerBuffer.get(cur).getValue(), value);
+ boolean notEOF = super.next((LongWritable) footerBuffer.get(cur).getKey(), (Text) footerBuffer.get(cur).getValue());
+ if (notEOF) {
+ cur = (++cur) % footerBuffer.size();
+ }
+ return notEOF;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/SkippingTextInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/SkippingTextInputFormat.java
new file mode 100644
index 0000000..7427aea
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/SkippingTextInputFormat.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * SkippingInputFormat is a header/footer aware input format. It truncates
+ * splits identified by TextInputFormat. Header and footers are removed
+ * from the splits.
+ *
+ * This InputFormat does NOT support Compressed Files!
+ */
+public class SkippingTextInputFormat extends TextInputFormat {
+
+ private final Map<Path, Long> startIndexMap = new ConcurrentHashMap<>();
+ private final Map<Path, Long> endIndexMap = new ConcurrentHashMap<>();
+ private JobConf conf;
+ private int headerCount;
+ private int footerCount;
+
+ @Override
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ super.configure(conf);
+ }
+
+ public void configure(JobConf conf, int headerCount, int footerCount) {
+ configure(conf);
+ this.headerCount = headerCount;
+ this.footerCount = footerCount;
+ }
+
+ @Override
+ protected FileSplit makeSplit(Path file, long start, long length, String[] hosts) {
+ return makeSplitInternal(file, start, length, hosts, null);
+ }
+
+ @Override
+ protected FileSplit makeSplit(Path file, long start, long length, String[] hosts, String[] inMemoryHosts) {
+ return makeSplitInternal(file, start, length, hosts, inMemoryHosts);
+ }
+
+ private FileSplit makeSplitInternal(Path file, long start, long length, String[] hosts, String[] inMemoryHosts) {
+ long cachedStart;
+ long cachedEnd;
+ try {
+ cachedStart = getCachedStartIndex(file);
+ cachedEnd = getCachedEndIndex(file);
+ } catch (IOException e) {
+ LOG.warn("Could not detect header/footer", e);
+ return new NullRowsInputFormat.DummyInputSplit(file);
+ }
+ if (cachedStart > start + length) {
+ return new NullRowsInputFormat.DummyInputSplit(file);
+ }
+ if (cachedStart > start) {
+ length = length - (cachedStart - start);
+ start = cachedStart;
+ }
+ if (cachedEnd < start) {
+ return new NullRowsInputFormat.DummyInputSplit(file);
+ }
+ if (cachedEnd < start + length) {
+ length = cachedEnd - start;
+ }
+ if (inMemoryHosts == null) {
+ return super.makeSplit(file, start, length, hosts);
+ } else {
+ return super.makeSplit(file, start, length, hosts, inMemoryHosts);
+ }
+ }
+
+ private long getCachedStartIndex(Path path) throws IOException {
+ if (headerCount == 0) {
+ return 0;
+ }
+ Long startIndexForFile = startIndexMap.get(path);
+ if (startIndexForFile == null) {
+ FileSystem fileSystem;
+ FSDataInputStream fis = null;
+ fileSystem = path.getFileSystem(conf);
+ try {
+ fis = fileSystem.open(path);
+ for (int j = 0; j < headerCount; j++) {
+ if (fis.readLine() == null) {
+ startIndexMap.put(path, Long.MAX_VALUE);
+ return Long.MAX_VALUE;
+ }
+ }
+ // Readers skip the entire first row if the start index of the
+ // split is not zero. We are setting the start of the index as
+ // the last byte of the previous row so the last line of header
+ // is discarded instead of the first valid input row.
+ startIndexForFile = fis.getPos() - 1;
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ startIndexMap.put(path, startIndexForFile);
+ }
+ return startIndexForFile;
+ }
+
+ private long getCachedEndIndex(Path path) throws IOException {
+ Long endIndexForFile = endIndexMap.get(path);
+ if (endIndexForFile == null) {
+ final long bufferSectionSize = 5 * 1024;
+ FileSystem fileSystem = path.getFileSystem(conf);
+ long endOfFile = fileSystem.getFileStatus(path).getLen();
+ if (footerCount == 0) {
+ endIndexForFile = endOfFile;
+ } else {
+ long bufferSectionEnd = endOfFile; // first byte that is not included in the section
+ long bufferSectionStart = Math.max(0, bufferSectionEnd - bufferSectionSize);
+
+ // we need 'footer count' lines and one space for EOF
+ LineBuffer buffer = new LineBuffer(footerCount + 1);
+ FSDataInputStream fis = null;
+ try {
+ fis = fileSystem.open(path);
+ while (bufferSectionEnd > bufferSectionStart) {
+ fis.seek(bufferSectionStart);
+ long pos = fis.getPos();
+ while (pos < bufferSectionEnd) {
+ if (fis.readLine() == null) {
+ // if there is not enough lines in this section, check the previous
+ // section. If this is the beginning section, there are simply not
+ // enough lines in the file.
+ break;
+ }
+ pos = fis.getPos();
+ buffer.consume(pos, bufferSectionEnd);
+ }
+ if (buffer.getRemainingLineCount() == 0) {
+ // if we consumed all the required line ends, that means the buffer now
+ // contains the index of the first byte of the footer.
+ break;
+ } else {
+ bufferSectionEnd = bufferSectionStart;
+ bufferSectionStart = Math.max(0, bufferSectionEnd - bufferSectionSize);
+ }
+ }
+ if (buffer.getRemainingLineCount() == 0) {
+ // buffer.getFirstLineStart() is the first byte of the footer. So the split
+ // must end before this.
+ endIndexForFile = buffer.getFirstLineStart() - 1;
+ } else {
+ // there were not enough lines in the file to consume all footer rows.
+ endIndexForFile = Long.MIN_VALUE;
+ }
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ }
+ endIndexMap.put(path, endIndexForFile);
+ }
+ return endIndexForFile;
+ }
+
+ static class LineBuffer {
+ private final Queue<Long> queue = new ArrayDeque<Long>();
+ private int remainingLineEnds;
+ private long lowPosition = Long.MAX_VALUE;
+
+ LineBuffer(int requiredLines) {
+ this.remainingLineEnds = requiredLines;
+ }
+
+ public void consume(long position, long sectionEnd) {
+ if (position > sectionEnd) {
+ return;
+ }
+ if (position < lowPosition) {
+ remainingLineEnds -= queue.size();
+ queue.clear();
+ queue.add(position);
+ lowPosition = position;
+ } else if (position > lowPosition) {
+ if (queue.size() == remainingLineEnds) {
+ queue.poll();
+ }
+ queue.add(position);
+ lowPosition = queue.peek();
+ }
+ }
+
+ public int getRemainingLineCount() {
+ return remainingLineEnds - queue.size();
+ }
+
+ public long getFirstLineStart() {
+ return lowPosition;
+ }
+ }
+}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
index 2a47abf..73a3728 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
@@ -59,7 +59,7 @@ public class TestHiveBinarySearchRecordReader extends TestCase {
private RCFileRecordReader rcfReader;
private JobConf conf;
private TestHiveInputSplit hiveSplit;
- private HiveContextAwareRecordReader hbsReader;
+ private HiveContextAwareRecordReader<WritableComparable, Writable> hbsReader;
private IOContext ioContext;
private static class TestHiveInputSplit extends HiveInputSplit {
@@ -148,52 +148,52 @@ private void init() throws IOException {
hbsReader.initIOContext(hiveSplit, conf, Class.class, rcfReader);
}
- private boolean executeDoNext(HiveContextAwareRecordReader hbsReader) throws IOException {
+ private boolean executeDoNext() throws IOException {
return hbsReader.next(hbsReader.createKey(), hbsReader.createValue());
}
public void testNonLinearGreaterThan() throws Exception {
init();
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(50);
ioContext.setComparison(1);
when(rcfReader.getPos()).thenReturn(25L);
// By setting the comparison to greater, the search should use the block [0, 50]
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(25);
}
public void testNonLinearLessThan() throws Exception {
init();
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(50);
ioContext.setComparison(-1);
when(rcfReader.getPos()).thenReturn(75L);
// By setting the comparison to less, the search should use the block [50, 100]
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(75);
}
public void testNonLinearEqualTo() throws Exception {
init();
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(50);
ioContext.setComparison(0);
when(rcfReader.getPos()).thenReturn(25L);
// By setting the comparison to equal, the search should use the block [0, 50]
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(25);
}
public void testHitLastBlock() throws Exception {
init();
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(50);
ioContext.setComparison(-1);
@@ -202,7 +202,7 @@ public void testHitLastBlock() throws Exception {
// When sync is called it will return 100, the value signaling the end of the file, this should
// result in a call to sync to the beginning of the block it was searching [50, 100], and it
// should continue normally
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
InOrder inOrder = inOrder(rcfReader);
inOrder.verify(rcfReader).sync(75);
inOrder.verify(rcfReader).sync(50);
@@ -211,14 +211,14 @@ public void testHitLastBlock() throws Exception {
public void testHitSamePositionTwice() throws Exception {
init();
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
verify(rcfReader).sync(50);
ioContext.setComparison(1);
// When getPos is called it should return the same value, signaling the end of the search, so
// the search should continue linearly and it should sync to the beginning of the block [0, 50]
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
InOrder inOrder = inOrder(rcfReader);
inOrder.verify(rcfReader).sync(25);
inOrder.verify(rcfReader).sync(0);
@@ -228,20 +228,20 @@ public void testHitSamePositionTwice() throws Exception {
public void testResetRange() throws Exception {
init();
InOrder inOrder = inOrder(rcfReader);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
inOrder.verify(rcfReader).sync(50);
ioContext.setComparison(-1);
when(rcfReader.getPos()).thenReturn(75L);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
inOrder.verify(rcfReader).sync(75);
ioContext.setEndBinarySearch(true);
// This should make the search linear, sync to the beginning of the block being searched
// [50, 100], set the comparison to be null, and the flag to reset the range should be unset
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
inOrder.verify(rcfReader).sync(50);
Assert.assertFalse(ioContext.isBinarySearching());
Assert.assertFalse(ioContext.shouldEndBinarySearch());
@@ -251,68 +251,68 @@ public void testEqualOpClass() throws Exception {
init();
ioContext.setGenericUDFClassName(GenericUDFOPEqual.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(0);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(1);
- Assert.assertFalse(executeDoNext(hbsReader));
+ Assert.assertFalse(executeDoNext());
}
public void testLessThanOpClass() throws Exception {
init();
ioContext.setGenericUDFClassName(GenericUDFOPLessThan.class.getName());
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
Assert.assertFalse(ioContext.isBinarySearching());
ioContext.setComparison(-1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(0);
- Assert.assertFalse(executeDoNext(hbsReader));
+ Assert.assertFalse(executeDoNext());
ioContext.setComparison(1);
- Assert.assertFalse(executeDoNext(hbsReader));
+ Assert.assertFalse(executeDoNext());
}
public void testLessThanOrEqualOpClass() throws Exception {
init();
ioContext.setGenericUDFClassName(GenericUDFOPEqualOrLessThan.class.getName());
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
Assert.assertFalse(ioContext.isBinarySearching());
ioContext.setComparison(-1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(0);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(1);
- Assert.assertFalse(executeDoNext(hbsReader));
+ Assert.assertFalse(executeDoNext());
}
public void testGreaterThanOpClass() throws Exception {
init();
ioContext.setGenericUDFClassName(GenericUDFOPGreaterThan.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(0);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
}
public void testGreaterThanOrEqualOpClass() throws Exception {
init();
ioContext.setGenericUDFClassName(GenericUDFOPEqualOrGreaterThan.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(0);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
ioContext.setComparison(1);
- Assert.assertTrue(executeDoNext(hbsReader));
+ Assert.assertTrue(executeDoNext());
}
public static void main(String[] args) throws Exception {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestLineBuffer.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestLineBuffer.java
new file mode 100644
index 0000000..71ca3ac
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestLineBuffer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * LineEndBuffer simple unit test.
+ */
+public class TestLineBuffer {
+
+ @Test
+ public void testLineEndBuffer() {
+ SkippingTextInputFormat.LineBuffer buffer;
+ buffer = new SkippingTextInputFormat.LineBuffer(3);
+ buffer.consume(200, 200);
+ buffer.consume(100, 200);
+ buffer.consume(100, 100);
+ buffer.consume(50, 100);
+ assertEquals(0, buffer.getRemainingLineCount());
+ assertEquals(50, buffer.getFirstLineStart());
+
+ buffer = new SkippingTextInputFormat.LineBuffer(3);
+ buffer.consume(200, 200);
+ buffer.consume(150, 200);
+ buffer.consume(100, 200);
+ assertEquals(0, buffer.getRemainingLineCount());
+ assertEquals(100, buffer.getFirstLineStart());
+
+ buffer = new SkippingTextInputFormat.LineBuffer(3);
+ buffer.consume(200, 200);
+ assertEquals(2, buffer.getRemainingLineCount());
+ buffer.consume(100, 100);
+ assertEquals(1, buffer.getRemainingLineCount());
+ buffer.consume(50, 100);
+ assertEquals(0, buffer.getRemainingLineCount());
+ assertEquals(50, buffer.getFirstLineStart());
+
+ buffer = new SkippingTextInputFormat.LineBuffer(3);
+ buffer.consume(200, 200);
+ assertEquals(2, buffer.getRemainingLineCount());
+ buffer.consume(50, 100);
+ assertEquals(1, buffer.getRemainingLineCount());
+ buffer.consume(25, 100);
+ assertEquals(0, buffer.getRemainingLineCount());
+ assertEquals(25, buffer.getFirstLineStart());
+ }
+}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestSkippingTextInputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestSkippingTextInputFormat.java
new file mode 100644
index 0000000..9607706
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestSkippingTextInputFormat.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unittest for SkippingTextInputFormat with Skip Header/Footer.
+ */
+public class TestSkippingTextInputFormat {
+
+ private Configuration conf;
+ private JobConf job;
+ private FileSystem fileSystem;
+ private Path testDir;
+ Reporter reporter;
+
+ private Path dataDir;
+
+ private CompressionCodecFactory compressionCodecs = null;
+ private CompressionCodec codec;
+
+ @Before
+ public void setUp() throws IOException {
+ conf = new Configuration();
+ job = new JobConf(conf);
+
+ TableDesc tblDesc = Utilities.defaultTd;
+ PartitionDesc partDesc = new PartitionDesc(tblDesc, null);
+ LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>();
+ pt.put(new Path("/tmp/testfolder"), partDesc);
+ MapredWork mrwork = new MapredWork();
+ mrwork.getMapWork().setPathToPartitionInfo(pt);
+ Utilities.setMapRedWork(job, mrwork,new Path("/tmp/" + System.getProperty("user.name"), "hive"));
+
+ fileSystem = FileSystem.getLocal(conf);
+ testDir = new Path(System.getProperty("test.tmp.dir", System.getProperty(
+ "user.dir", new File(".").getAbsolutePath()))
+ + "/TestSkippingTextInputFormat");
+ reporter = Reporter.NULL;
+ fileSystem.delete(testDir, true);
+
+ dataDir = new Path(testDir, "datadir");
+ fileSystem.mkdirs(dataDir);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ fileSystem.delete(testDir, true);
+ }
+
+ /**
+ * Test CSV input file with header/footer skip.
+ */
+ @Test
+ public void testSkipFileSplits() throws Exception {
+ FileSystem fs = dataDir.getFileSystem(job);
+ FileInputFormat.setInputPaths(job, dataDir);
+
+ // First Dir with 1 File
+ Path dir1_file1 = new Path(dataDir, "skipfile1.csv");
+ writeTextFile(dir1_file1,
+ "dir1_header\n" +
+ "dir1_file1_line1\n" +
+ "dir1_file1_line2\n" +
+ "dir1_footer"
+ );
+
+ SkippingTextInputFormat inputFormat = new SkippingTextInputFormat();
+ // One header and one footer line to be deducted
+ inputFormat.configure(job, 1, 1);
+
+ FileInputFormat.setInputPaths(job, dir1_file1);
+ InputSplit[] splits = inputFormat.getSplits(job, 2);
+
+ assertTrue(splits.length == 2);
+
+ // Read all values.
+ List<String> received = new ArrayList<String>();
+ for (int i=0; i < splits.length; i++) {
+ RecordReader<LongWritable, Text> reader =
+ inputFormat.getRecordReader(splits[i], job, reporter);
+
+ HiveInputFormat.HiveInputSplit hiveInputSplit =
+ new HiveInputFormat.HiveInputSplit(splits[i], inputFormat.getClass().getName());
+ assertTrue(hiveInputSplit.getInputSplit().getClass() == FileSplit.class);
+
+ LongWritable key = reader.createKey();
+ Text value = reader.createValue();
+ while (reader.next(key, value)) {
+ received.add(value.toString());
+ }
+ reader.close();
+ }
+ // make sure we skipped the header and the footer across splits
+ assertTrue(received.size() == 2);
+ assertTrue(!received.get(0).contains("header"));
+ assertTrue(!received.get(received.size()-1).contains("footer"));
+ }
+
+ /**
+ * Test compressed CSV input file with header/footer skip.
+ */
+ @Test
+ public void testSkipCompressedFileSplits() throws Exception {
+ FileSystem fs = dataDir.getFileSystem(job);
+ FileInputFormat.setInputPaths(job, dataDir);
+
+ // First Dir with 1 Compressed CSV File
+ Path dir1_file1 = new Path(dataDir, "skipfile1.csv.bz2");
+ writeTextFile(dir1_file1,
+ "dir1_header\n" +
+ "dir1_file1_line1\n" +
+ "dir1_file1_line2\n" +
+ "dir1_footer"
+ );
+
+ SkippingTextInputFormat inputFormat = new SkippingTextInputFormat();
+ // One header and one footer line to be deducted
+ inputFormat.configure(job, 1, 1);
+
+ compressionCodecs = new CompressionCodecFactory(conf);
+ codec = compressionCodecs.getCodec(dir1_file1);
+ System.out.println("Codec: "+ codec);
+
+ FileInputFormat.setInputPaths(job, dir1_file1);
+ InputSplit[] splits = inputFormat.getSplits(job, 1);
+
+ // Should not generate splits for compressed file!
+ assertTrue(splits.length == 1);
+
+ // Read all values.
+ List<String> received = new ArrayList<String>();
+ for (int i=0; i < splits.length; i++) {
+ RecordReader<LongWritable, Text> reader =
+ inputFormat.getRecordReader(splits[i], job, reporter);
+
+ HiveInputFormat.HiveInputSplit hiveInputSplit =
+ new HiveInputFormat.HiveInputSplit(splits[i], inputFormat.getClass().getName());
+ System.out.println(hiveInputSplit.getInputSplit().getClass());
+ assertTrue(FileSplit.class == hiveInputSplit.getInputSplit().getClass());
+ System.out.println("Split: [" +i + "] "+ hiveInputSplit.getStart() + " => " + hiveInputSplit.getLength());
+
+ LongWritable key = reader.createKey();
+ Text value = reader.createValue();
+ while (reader.next(key, value)) {
+ System.out.println("Splits:" + i + " Val: "+ value);
+ received.add(value.toString());
+ }
+ reader.close();
+ }
+ }
+
+ /**
+ * Writes the given string to the given file.
+ */
+ private void writeTextFile(Path file, String content) throws IOException {
+ OutputStreamWriter writer = new OutputStreamWriter(fileSystem.create(file));
+ writer.write(content);
+ writer.close();
+ }
+}
diff --git a/ql/src/test/queries/clientpositive/compressed_skip_header_footer_aggr.q b/ql/src/test/queries/clientpositive/compressed_skip_header_footer_aggr.q
new file mode 100644
index 0000000..58853be
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/compressed_skip_header_footer_aggr.q
@@ -0,0 +1,100 @@
+SET hive.query.results.cache.enabled=false;
+SET hive.mapred.mode=nonstrict;
+SET hive.explain.user=false;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/testcase1;
+dfs -copyFromLocal ../../data/files/compressed_4line_file1.csv ${system:test.tmp.dir}/testcase1/;
+--
+--
+CREATE EXTERNAL TABLE `testcase1`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase1'
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1");
+
+CREATE EXTERNAL TABLE `testcase2`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase1'
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0");
+
+CREATE EXTERNAL TABLE `testcase3`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase1'
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1");
+
+SET hive.fetch.task.conversion = more;
+
+select * from testcase1;
+select count(*) from testcase1;
+
+select * from testcase2;
+select count(*) from testcase2;
+
+select * from testcase3;
+select count(*) from testcase3;
+
+SET hive.fetch.task.conversion = none;
+
+
+select * from testcase1;
+select count(*) from testcase1;
+
+select * from testcase2;
+select count(*) from testcase2;
+
+select * from testcase3;
+select count(*) from testcase3;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/testcase2;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/testcase3;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/testcase4;
+dfs -copyFromLocal ../../data/files/compressed_4line_file2.csv.bz2 ${system:test.tmp.dir}/testcase2/;
+dfs -copyFromLocal ../../data/files/compressed_4line_file2.csv.bz2 ${system:test.tmp.dir}/testcase3/;
+dfs -copyFromLocal ../../data/files/compressed_4line_file2.csv.bz2 ${system:test.tmp.dir}/testcase4/;
+--
+-- Stored encoded in Cache so need to create separate Tables
+CREATE EXTERNAL TABLE `testcase4`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase2'
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1");
+
+CREATE EXTERNAL TABLE `testcase5`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase3'
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0");
+
+CREATE EXTERNAL TABLE `testcase6`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+ LOCATION '${system:test.tmp.dir}/testcase4'
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1");
+
+SET hive.fetch.task.conversion = more;
+
+select * from testcase4;
+select count(*) from testcase4;
+
+select * from testcase5;
+select count(*) from testcase5;
+
+select * from testcase6;
+select count(*) from testcase6;
+
+SET hive.fetch.task.conversion = none;
+
+select * from testcase4;
+select count(*) from testcase4;
+
+select * from testcase5;
+select count(*) from testcase5;
+
+select * from testcase6;
+select count(*) from testcase6;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/testcase_gz;
+dfs -copyFromLocal ../../data/files/test.csv.gz ${system:test.tmp.dir}/testcase_gz/;
+
+create table testcase_gz(age int, name string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+stored as textfile LOCATION '${system:test.tmp.dir}/testcase_gz'
+TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1");
+
+SET hive.fetch.task.conversion = more;
+select * from testcase_gz;
+select count(*) from testcase_gz;
+
+set hive.fetch.task.conversion=none;
+select * from testcase_gz;
+select count(*) from testcase_gz;
\ No newline at end of file
diff --git a/ql/src/test/queries/clientpositive/skip_header_footer_aggr.q b/ql/src/test/queries/clientpositive/skip_header_footer_aggr.q
new file mode 100644
index 0000000..b98da38
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/skip_header_footer_aggr.q
@@ -0,0 +1,98 @@
+set hive.mapred.mode=nonstrict;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir};
+dfs -copyFromLocal ../../data/files/header_footer_table_4 ${system:test.tmp.dir}/header_footer_table_4;
+
+CREATE TABLE numbrs (numbr int);
+INSERT INTO numbrs VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12), (NULL);
+CREATE EXTERNAL TABLE header_footer_table_4 (header_int int, header_name string, header_choice varchar(10)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LOCATION '${system:test.tmp.dir}/header_footer_table_4' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="2");
+
+SELECT * FROM header_footer_table_4;
+
+SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8;
+
+-- should return nothing as title is correctly skipped
+SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice';
+SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly';
+SELECT COUNT(*) FROM header_footer_table_4;
+
+-- shpuld return nothing
+SELECT * FROM header_footer_table_4 WHERE header_choice is NULL;
+
+
+SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+-- should not include any header and footer
+SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int;
+
+-- no join variant should include header or footer
+SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int;
+
+
+SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+
+SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+
+SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8;
+
+-- create table from existing text file should not feed header or footer to the new table
+CREATE TABLE transition (title VARCHAR(10), name VARCHAR(10)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE LOCATION '${system:test.tmp.dir}/header_footer_table_transition';
+INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4;
+SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name;
+
+CREATE TABLE transition2 (header_choice VARCHAR(10), sum_header_int int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE LOCATION '${system:test.tmp.dir}/test/header_footer_table_transition2';
+INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice;
+
+DROP TABLE transition;
+DROP TABLE transition2;
+
+-- turn off fetch conversion. This disables the additional header/footer handling logic in fetch operator.
+set hive.fetch.task.conversion=none;
+
+SELECT * FROM header_footer_table_4;
+
+SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8;
+
+-- should return nothing as title is correctly skipped
+SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice';
+SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly';
+SELECT COUNT(*) FROM header_footer_table_4;
+
+-- shpuld return nothing
+SELECT * FROM header_footer_table_4 WHERE header_choice is NULL;
+
+
+SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+-- should not include any header and footer
+SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int;
+
+-- no join variant should include header or footer
+SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int;
+SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int;
+
+
+SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+
+SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+
+
+SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8;
+
+-- create table from existing text file should not feed header or footer to the new table
+CREATE TABLE transition (title VARCHAR(10), name VARCHAR(10)) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE LOCATION '${system:test.tmp.dir}/header_footer_table_transition';
+INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4;
+SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name;
+
+CREATE TABLE transition2 (header_choice VARCHAR(10), sum_header_int int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE LOCATION '${system:test.tmp.dir}/test/header_footer_table_transition2';
+INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice;
+SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice;
diff --git a/ql/src/test/queries/clientpositive/skip_header_footer_proj.q b/ql/src/test/queries/clientpositive/skip_header_footer_proj.q
new file mode 100644
index 0000000..478baef
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/skip_header_footer_proj.q
@@ -0,0 +1,25 @@
+SET hive.fetch.task.conversion=none;
+-- no footer
+DROP TABLE IF EXISTS hf1;
+CREATE TABLE hf1 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1');
+INSERT OVERWRITE TABLE hf1 VALUES ('x','y'),('a','b'),('c','d');
+SELECT * FROM hf1;
+
+-- no header
+DROP TABLE IF EXISTS hf2;
+CREATE TABLE hf2 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.footer.line.count'='2');
+INSERT OVERWRITE TABLE hf2 VALUES ('x','y'),('a','b'),('c','d');
+SELECT * FROM hf2;
+
+-- only header, no data
+DROP TABLE IF EXISTS hf3;
+CREATE TABLE hf3 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='3');
+INSERT OVERWRITE TABLE hf3 VALUES ('x','y'),('a','b'),('c','d');
+SELECT * FROM hf3;
+
+-- header and footer, no data
+DROP TABLE IF EXISTS hf4;
+CREATE TABLE hf4 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1','skip.footer.line.count'='2');
+INSERT OVERWRITE TABLE hf4 VALUES ('x','y'),('a','b'),('c','d');
+SELECT * FROM hf4;
+
diff --git a/ql/src/test/results/clientpositive/llap/compressed_skip_header_footer_aggr.q.out b/ql/src/test/results/clientpositive/llap/compressed_skip_header_footer_aggr.q.out
new file mode 100644
index 0000000..faea853
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/compressed_skip_header_footer_aggr.q.out
@@ -0,0 +1,361 @@
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase1`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase1
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase1`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase1
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase2`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase2
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase2`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase2
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase3`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase3
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase3`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase3
+PREHOOK: query: select * from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+3 2019-12-31
+PREHOOK: query: select count(*) from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase4`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase4
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase4`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase4
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase5`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase5
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase5`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase5
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase6`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase6
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase6`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase6
+PREHOOK: query: select * from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+3 2019-12-31 02
+PREHOOK: query: select count(*) from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31 00
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+3 2019-12-31 02
+PREHOOK: query: select count(*) from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31 00
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: create table testcase_gz(age int, name string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase_gz
+POSTHOOK: query: create table testcase_gz(age int, name string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase_gz
+PREHOOK: query: select * from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a
+2 b
+PREHOOK: query: select count(*) from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a
+2 b
+PREHOOK: query: select count(*) from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
diff --git a/ql/src/test/results/clientpositive/llap/create_genericudaf.q.out b/ql/src/test/results/clientpositive/llap/create_genericudaf.q.out
new file mode 100644
index 0000000..d71de35
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/create_genericudaf.q.out
@@ -0,0 +1,108 @@
+PREHOOK: query: EXPLAIN
+CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_avg
+POSTHOOK: query: EXPLAIN
+CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_avg
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_avg
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_avg
+PREHOOK: query: EXPLAIN
+SELECT
+ test_avg(1),
+ test_avg(substr(value,5))
+FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: EXPLAIN
+SELECT
+ test_avg(1),
+ test_avg(substr(value,5))
+FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: substr(value, 5) (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: test_avg(1), test_avg(_col1)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: struct<count:bigint,sum:double,input:string>)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: test_avg(VALUE._col0), test_avg(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ test_avg(1),
+ test_avg(substr(value,5))
+FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT
+ test_avg(1),
+ test_avg(substr(value,5))
+FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1.0 260.182
+PREHOOK: query: DROP TEMPORARY FUNCTIOn test_avg
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: test_avg
+POSTHOOK: query: DROP TEMPORARY FUNCTIOn test_avg
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: test_avg
diff --git a/ql/src/test/results/clientpositive/llap/create_udaf.q.out b/ql/src/test/results/clientpositive/llap/create_udaf.q.out
new file mode 100644
index 0000000..b2dd85a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/create_udaf.q.out
@@ -0,0 +1,97 @@
+PREHOOK: query: EXPLAIN
+CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_max
+POSTHOOK: query: EXPLAIN
+CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_max
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_max
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_max
+PREHOOK: query: CREATE TABLE dest1_n34(col INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1_n34
+POSTHOOK: query: CREATE TABLE dest1_n34(col INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1_n34
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n34 SELECT test_max(length(src.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1_n34
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n34 SELECT test_max(length(src.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1_n34
+POSTHOOK: Lineage: dest1_n34.col EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT dest1_n34.* FROM dest1_n34
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1_n34
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT dest1_n34.* FROM dest1_n34
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1_n34
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7
+PREHOOK: query: SELECT test_max(CAST(length(src.value) AS SMALLINT)) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS SMALLINT)) FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7
+PREHOOK: query: SELECT test_max(CAST(length(src.value) AS BIGINT)) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS BIGINT)) FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7
+PREHOOK: query: SELECT test_max(CAST(length(src.value) AS DOUBLE)) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS DOUBLE)) FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7.0
+PREHOOK: query: SELECT test_max(CAST(length(src.value) AS FLOAT)) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS FLOAT)) FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7.0
+PREHOOK: query: SELECT test_max(substr(src.value,5)) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT test_max(substr(src.value,5)) FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+98
+PREHOOK: query: DROP TEMPORARY FUNCTION test_max
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: test_max
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_max
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: test_max
diff --git a/ql/src/test/results/clientpositive/llap/create_view.q.out b/ql/src/test/results/clientpositive/llap/create_view.q.out
new file mode 100644
index 0000000..d77e0f2
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/create_view.q.out
@@ -0,0 +1,1766 @@
+PREHOOK: query: DROP VIEW view1
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view1
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view2
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view2
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view3
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view3
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view4
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view4
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view5
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view5
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view6
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view6
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view7
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view7
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view8
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view8
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view9
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view9
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view10
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view10
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view11
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view11
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view12
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view12
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view13
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view13
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view14
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view14
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view15
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view15
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP VIEW view16
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view16
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP TEMPORARY FUNCTION test_translate
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_translate
+POSTHOOK: type: DROPFUNCTION
+PREHOOK: query: DROP TEMPORARY FUNCTION test_max
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_max
+POSTHOOK: type: DROPFUNCTION
+PREHOOK: query: DROP TEMPORARY FUNCTION test_explode
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_explode
+POSTHOOK: type: DROPFUNCTION
+PREHOOK: query: SELECT * FROM src WHERE key=86
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM src WHERE key=86
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86 val_86
+PREHOOK: query: CREATE VIEW view1 AS SELECT value FROM src WHERE key=86
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view1
+POSTHOOK: query: CREATE VIEW view1 AS SELECT value FROM src WHERE key=86
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view1
+POSTHOOK: Lineage: view1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE VIEW view2 AS SELECT * FROM src
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view2
+POSTHOOK: query: CREATE VIEW view2 AS SELECT * FROM src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view2
+POSTHOOK: Lineage: view2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE VIEW view3(valoo)
+TBLPROPERTIES ("fear" = "factor")
+AS SELECT upper(value) FROM src WHERE key=86
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view3
+POSTHOOK: query: CREATE VIEW view3(valoo)
+TBLPROPERTIES ("fear" = "factor")
+AS SELECT upper(value) FROM src WHERE key=86
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view3
+POSTHOOK: Lineage: view3.valoo EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * from view1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * from view1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+val_86
+PREHOOK: query: SELECT * from view2 where key=18
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * from view2 where key=18
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+18 val_18
+18 val_18
+PREHOOK: query: SELECT * from view3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * from view3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+VAL_86
+PREHOOK: query: EXPLAIN
+CREATE VIEW view0(valoo) AS SELECT upper(value) FROM src WHERE key=86
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view0
+POSTHOOK: query: EXPLAIN
+CREATE VIEW view0(valoo) AS SELECT upper(value) FROM src WHERE key=86
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view0
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Create View Operator:
+ Create View
+ columns: valoo string
+ expanded text: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `view0`
+ name: default.view0
+ original text: SELECT upper(value) FROM src WHERE key=86
+
+PREHOOK: query: EXPLAIN
+SELECT * from view2 where key=18
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: EXPLAIN
+SELECT * from view2 where key=18
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: src
+ properties:
+ insideView TRUE
+ Filter Operator
+ predicate: (UDFToDouble(key) = 18.0D) (type: boolean)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ ListSink
+
+PREHOOK: query: SHOW TABLES 'view.*'
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES 'view.*'
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+view1
+view2
+view3
+PREHOOK: query: SHOW VIEWS 'view.*'
+PREHOOK: type: SHOWVIEWS
+POSTHOOK: query: SHOW VIEWS 'view.*'
+POSTHOOK: type: SHOWVIEWS
+view1
+view2
+view3
+PREHOOK: query: DESCRIBE view1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view1
+POSTHOOK: query: DESCRIBE view1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view1
+value string
+PREHOOK: query: DESCRIBE EXTENDED view1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view1
+POSTHOOK: query: DESCRIBE EXTENDED view1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view1
+value string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view1
+POSTHOOK: query: DESCRIBE FORMATTED view1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view1
+# col_name data_type comment
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT value FROM src WHERE key=86
+Expanded Query: SELECT `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86
+PREHOOK: query: DESCRIBE view2
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view2
+POSTHOOK: query: DESCRIBE view2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view2
+key string
+value string
+PREHOOK: query: DESCRIBE EXTENDED view2
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view2
+POSTHOOK: query: DESCRIBE EXTENDED view2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view2
+key string
+value string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view2
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view2
+POSTHOOK: query: DESCRIBE FORMATTED view2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view2
+# col_name data_type comment
+key string
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT * FROM src
+Expanded Query: SELECT `src`.`key`, `src`.`value` FROM `default`.`src`
+PREHOOK: query: DESCRIBE view3
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view3
+POSTHOOK: query: DESCRIBE view3
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view3
+valoo string
+PREHOOK: query: DESCRIBE EXTENDED view3
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view3
+POSTHOOK: query: DESCRIBE EXTENDED view3
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view3
+valoo string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view3
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view3
+POSTHOOK: query: DESCRIBE FORMATTED view3
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view3
+# col_name data_type comment
+valoo string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+ fear factor
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT upper(value) FROM src WHERE key=86
+Expanded Query: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `view3`
+PREHOOK: query: ALTER VIEW view3 SET TBLPROPERTIES ("biggest" = "loser")
+PREHOOK: type: ALTERVIEW_PROPERTIES
+PREHOOK: Input: default@view3
+PREHOOK: Output: default@view3
+POSTHOOK: query: ALTER VIEW view3 SET TBLPROPERTIES ("biggest" = "loser")
+POSTHOOK: type: ALTERVIEW_PROPERTIES
+POSTHOOK: Input: default@view3
+POSTHOOK: Output: default@view3
+PREHOOK: query: DESCRIBE EXTENDED view3
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view3
+POSTHOOK: query: DESCRIBE EXTENDED view3
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view3
+valoo string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view3
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view3
+POSTHOOK: query: DESCRIBE FORMATTED view3
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view3
+# col_name data_type comment
+valoo string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ biggest loser
+ bucketing_version 2
+ fear factor
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT upper(value) FROM src WHERE key=86
+Expanded Query: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `view3`
+PREHOOK: query: CREATE TABLE table1_n4 (key int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1_n4
+POSTHOOK: query: CREATE TABLE table1_n4 (key int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1_n4
+PREHOOK: query: DESCRIBE EXTENDED table1_n4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@table1_n4
+POSTHOOK: query: DESCRIBE EXTENDED table1_n4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@table1_n4
+key int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE EXTENDED src1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src1
+POSTHOOK: query: DESCRIBE EXTENDED src1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src1
+key string default
+value string default
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE EXTENDED table1_n4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@table1_n4
+POSTHOOK: query: DESCRIBE EXTENDED table1_n4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@table1_n4
+key int
+
+#### A masked pattern was here ####
+PREHOOK: query: INSERT OVERWRITE TABLE table1_n4 SELECT key FROM src WHERE key = 86
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@table1_n4
+POSTHOOK: query: INSERT OVERWRITE TABLE table1_n4 SELECT key FROM src WHERE key = 86
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@table1_n4
+POSTHOOK: Lineage: table1_n4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table1_n4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM table1_n4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86
+PREHOOK: query: CREATE VIEW view4 AS SELECT * FROM table1_n4
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view4
+POSTHOOK: query: CREATE VIEW view4 AS SELECT * FROM table1_n4
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view4
+POSTHOOK: Lineage: view4.key SIMPLE [(table1_n4)table1_n4.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: SELECT * FROM view4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86
+PREHOOK: query: DESCRIBE view4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view4
+POSTHOOK: query: DESCRIBE view4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view4
+key int
+PREHOOK: query: ALTER TABLE table1_n4 ADD COLUMNS (value STRING)
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: default@table1_n4
+POSTHOOK: query: ALTER TABLE table1_n4 ADD COLUMNS (value STRING)
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: default@table1_n4
+PREHOOK: query: SELECT * FROM table1_n4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM table1_n4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86 NULL
+PREHOOK: query: SELECT * FROM view4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86
+PREHOOK: query: DESCRIBE table1_n4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@table1_n4
+POSTHOOK: query: DESCRIBE table1_n4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@table1_n4
+key int
+value string
+PREHOOK: query: DESCRIBE view4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view4
+POSTHOOK: query: DESCRIBE view4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view4
+key int
+PREHOOK: query: CREATE VIEW view5 AS SELECT v1.key as key1, v2.key as key2
+FROM view4 v1 join view4 v2
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view5
+POSTHOOK: query: CREATE VIEW view5 AS SELECT v1.key as key1, v2.key as key2
+FROM view4 v1 join view4 v2
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view5
+POSTHOOK: Lineage: view5.key1 SIMPLE [(table1_n4)table1_n4.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: view5.key2 SIMPLE [(table1_n4)table1_n4.FieldSchema(name:key, type:int, comment:null), ]
+Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM view5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view4
+PREHOOK: Input: default@view5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view4
+POSTHOOK: Input: default@view5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86 86
+PREHOOK: query: DESCRIBE view5
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view5
+POSTHOOK: query: DESCRIBE view5
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view5
+key1 int
+key2 int
+PREHOOK: query: CREATE VIEW view6(valoo COMMENT 'I cannot spell') AS
+SELECT upper(value) as blarg FROM src WHERE key=86
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view6
+POSTHOOK: query: CREATE VIEW view6(valoo COMMENT 'I cannot spell') AS
+SELECT upper(value) as blarg FROM src WHERE key=86
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view6
+POSTHOOK: Lineage: view6.valoo EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE view6
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view6
+POSTHOOK: query: DESCRIBE view6
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view6
+valoo string I cannot spell
+PREHOOK: query: CREATE VIEW view7 AS
+SELECT * FROM src
+WHERE key > 80 AND key < 100
+ORDER BY key, value
+LIMIT 10
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view7
+POSTHOOK: query: CREATE VIEW view7 AS
+SELECT * FROM src
+WHERE key > 80 AND key < 100
+ORDER BY key, value
+LIMIT 10
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view7
+POSTHOOK: Lineage: view7.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view7.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM view7
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view7
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view7
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view7
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+82 val_82
+83 val_83
+83 val_83
+84 val_84
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+90 val_90
+90 val_90
+PREHOOK: query: SELECT * FROM view7 ORDER BY key DESC, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view7
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view7 ORDER BY key DESC, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view7
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+90 val_90
+90 val_90
+87 val_87
+86 val_86
+85 val_85
+84 val_84
+84 val_84
+83 val_83
+83 val_83
+82 val_82
+PREHOOK: query: SELECT * FROM view7 LIMIT 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view7
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view7 LIMIT 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view7
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+82 val_82
+83 val_83
+83 val_83
+84 val_84
+84 val_84
+PREHOOK: query: SELECT * FROM view7 LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view7
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view7 LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view7
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+82 val_82
+83 val_83
+83 val_83
+84 val_84
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+90 val_90
+90 val_90
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_translate AS
+'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestTranslate'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_translate
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_translate AS
+'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestTranslate'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_translate
+PREHOOK: query: CREATE VIEW view8(c) AS
+SELECT test_translate('abc', 'a', 'b')
+FROM table1_n4
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view8
+POSTHOOK: query: CREATE VIEW view8(c) AS
+SELECT test_translate('abc', 'a', 'b')
+FROM table1_n4
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view8
+POSTHOOK: Lineage: view8.c SIMPLE []
+PREHOOK: query: DESCRIBE EXTENDED view8
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view8
+POSTHOOK: query: DESCRIBE EXTENDED view8
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view8
+c string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view8
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view8
+POSTHOOK: query: DESCRIBE FORMATTED view8
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view8
+# col_name data_type comment
+c string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT test_translate('abc', 'a', 'b')
+ FROM table1_n4
+Expanded Query: SELECT `_c0` AS `c` FROM (SELECT `test_translate`('abc', 'a', 'b')
+ FROM `default`.`table1_n4`) `view8`
+PREHOOK: query: SELECT * FROM view8
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view8
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view8
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view8
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+bbc
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_max AS
+'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_max
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_max AS
+'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_max
+PREHOOK: query: CREATE VIEW view9(m) AS
+SELECT test_max(length(value))
+FROM src
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view9
+POSTHOOK: query: CREATE VIEW view9(m) AS
+SELECT test_max(length(value))
+FROM src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view9
+POSTHOOK: Lineage: view9.m EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view9
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view9
+POSTHOOK: query: DESCRIBE EXTENDED view9
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view9
+m int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view9
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view9
+POSTHOOK: query: DESCRIBE FORMATTED view9
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view9
+# col_name data_type comment
+m int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT test_max(length(value))
+ FROM src
+Expanded Query: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`))
+ FROM `default`.`src`) `view9`
+PREHOOK: query: SELECT * FROM view9
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view9
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view9
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view9
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7
+PREHOOK: query: DROP VIEW view9
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view9
+PREHOOK: Output: default@view9
+POSTHOOK: query: DROP VIEW view9
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view9
+POSTHOOK: Output: default@view9
+PREHOOK: query: CREATE VIEW view9(m) AS
+SELECT test_max(length(value))
+FROM src
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view9
+POSTHOOK: query: CREATE VIEW view9(m) AS
+SELECT test_max(length(value))
+FROM src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view9
+POSTHOOK: Lineage: view9.m EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view9
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view9
+POSTHOOK: query: DESCRIBE EXTENDED view9
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view9
+m int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view9
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view9
+POSTHOOK: query: DESCRIBE FORMATTED view9
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view9
+# col_name data_type comment
+m int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT test_max(length(value))
+ FROM src
+Expanded Query: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`))
+ FROM `default`.`src`) `view9`
+PREHOOK: query: SELECT * FROM view9
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view9
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view9
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view9
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+7
+PREHOOK: query: CREATE VIEW view10 AS
+SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view10
+POSTHOOK: query: CREATE VIEW view10 AS
+SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view10
+POSTHOOK: Lineage: view10.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view10.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view10
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view10
+POSTHOOK: query: DESCRIBE EXTENDED view10
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view10
+key string
+value string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view10
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view10
+POSTHOOK: query: DESCRIBE FORMATTED view10
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view10
+# col_name data_type comment
+key string
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp
+Expanded Query: SELECT `slurp`.`key`, `slurp`.`value` FROM (SELECT `src`.`key`, `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86) `slurp`
+PREHOOK: query: SELECT * FROM view10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view10
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view10
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+86 val_86
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_explode AS
+'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_explode
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_explode AS
+'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_explode
+PREHOOK: query: CREATE VIEW view11 AS
+SELECT test_explode(array(1,2,3)) AS (boom)
+FROM table1_n4
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view11
+POSTHOOK: query: CREATE VIEW view11 AS
+SELECT test_explode(array(1,2,3)) AS (boom)
+FROM table1_n4
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view11
+POSTHOOK: Lineage: view11.boom SCRIPT []
+PREHOOK: query: DESCRIBE EXTENDED view11
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view11
+POSTHOOK: query: DESCRIBE EXTENDED view11
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view11
+boom int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view11
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view11
+POSTHOOK: query: DESCRIBE FORMATTED view11
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view11
+# col_name data_type comment
+boom int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT test_explode(array(1,2,3)) AS (boom)
+ FROM table1_n4
+Expanded Query: SELECT `test_explode`(array(1,2,3)) AS (`boom`)
+ FROM `default`.`table1_n4`
+PREHOOK: query: SELECT * FROM view11
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1_n4
+PREHOOK: Input: default@view11
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view11
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Input: default@view11
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+2
+3
+PREHOOK: query: CREATE VIEW view12 AS
+SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view12
+POSTHOOK: query: CREATE VIEW view12 AS
+SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view12
+POSTHOOK: Lineage: view12.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view12.mycol SCRIPT []
+POSTHOOK: Lineage: view12.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view12
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view12
+POSTHOOK: query: DESCRIBE EXTENDED view12
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view12
+key string
+value string
+mycol int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view12
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view12
+POSTHOOK: query: DESCRIBE FORMATTED view12
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view12
+# col_name data_type comment
+key string
+value string
+mycol int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol
+Expanded Query: SELECT `src`.`key`, `src`.`value`, `mytable`.`mycol` FROM `default`.`src` LATERAL VIEW explode(array(1,2,3)) `myTable` AS `myCol`
+PREHOOK: query: SELECT * FROM view12
+ORDER BY key ASC, myCol ASC LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view12
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view12
+ORDER BY key ASC, myCol ASC LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view12
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 val_0 1
+PREHOOK: query: SELECT * FROM view2 LATERAL VIEW explode(array(1,2,3)) myTable AS myCol
+ORDER BY key ASC, myCol ASC LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view2 LATERAL VIEW explode(array(1,2,3)) myTable AS myCol
+ORDER BY key ASC, myCol ASC LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 val_0 1
+PREHOOK: query: CREATE VIEW view13 AS
+SELECT s.key
+FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@srcbucket
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view13
+POSTHOOK: query: CREATE VIEW view13 AS
+SELECT s.key
+FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@srcbucket
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view13
+POSTHOOK: Lineage: view13.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: DESCRIBE EXTENDED view13
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view13
+POSTHOOK: query: DESCRIBE EXTENDED view13
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view13
+key int
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view13
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view13
+POSTHOOK: query: DESCRIBE FORMATTED view13
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view13
+# col_name data_type comment
+key int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT s.key
+ FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s
+Expanded Query: SELECT `s`.`key`
+ FROM `default`.`srcbucket` TABLESAMPLE (BUCKET 1 OUT OF 5 ON `key`) `s`
+PREHOOK: query: SELECT * FROM view13
+ORDER BY key LIMIT 12
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket
+PREHOOK: Input: default@view13
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view13
+ORDER BY key LIMIT 12
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket
+POSTHOOK: Input: default@view13
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+8
+8
+16
+16
+18
+18
+21
+21
+21
+21
+24
+PREHOOK: query: CREATE VIEW view14 AS
+SELECT unionsrc1.key as k1, unionsrc1.value as v1,
+ unionsrc2.key as k2, unionsrc2.value as v2
+FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+ UNION ALL
+ select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1
+JOIN
+ (select 'tst1' as key, cast(count(1) as string) as value from src s3
+ UNION ALL
+ select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2
+ON (unionsrc1.key = unionsrc2.key)
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view14
+POSTHOOK: query: CREATE VIEW view14 AS
+SELECT unionsrc1.key as k1, unionsrc1.value as v1,
+ unionsrc2.key as k2, unionsrc2.value as v2
+FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+ UNION ALL
+ select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1
+JOIN
+ (select 'tst1' as key, cast(count(1) as string) as value from src s3
+ UNION ALL
+ select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2
+ON (unionsrc1.key = unionsrc2.key)
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view14
+POSTHOOK: Lineage: view14.k1 EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view14.k2 EXPRESSION [(src)s4.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view14.v1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: view14.v2 EXPRESSION [(src)s3.null, (src)s4.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view14
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view14
+POSTHOOK: query: DESCRIBE EXTENDED view14
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view14
+k1 string
+v1 string
+k2 string
+v2 string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view14
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view14
+POSTHOOK: query: DESCRIBE FORMATTED view14
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view14
+# col_name data_type comment
+k1 string
+v1 string
+k2 string
+v2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT unionsrc1.key as k1, unionsrc1.value as v1,
+ unionsrc2.key as k2, unionsrc2.value as v2
+ FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+ UNION ALL
+ select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1
+ JOIN
+ (select 'tst1' as key, cast(count(1) as string) as value from src s3
+ UNION ALL
+ select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2
+ ON (unionsrc1.key = unionsrc2.key)
+Expanded Query: SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v1`,
+ `unionsrc2`.`key` as `k2`, `unionsrc2`.`value` as `v2`
+ FROM (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s1`
+ UNION ALL
+ select `s2`.`key` as `key`, `s2`.`value` as `value` from `default`.`src` `s2` where `s2`.`key` < 10) `unionsrc1`
+ JOIN
+ (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s3`
+ UNION ALL
+ select `s4`.`key` as `key`, `s4`.`value` as `value` from `default`.`src` `s4` where `s4`.`key` < 10) `unionsrc2`
+ ON (`unionsrc1`.`key` = `unionsrc2`.`key`)
+PREHOOK: query: SELECT * FROM view14
+ORDER BY k1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view14
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view14
+ORDER BY k1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view14
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+2 val_2 2 val_2
+4 val_4 4 val_4
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+8 val_8 8 val_8
+9 val_9 9 val_9
+tst1 500 tst1 500
+PREHOOK: query: CREATE VIEW view15 AS
+SELECT key,COUNT(value) AS value_count
+FROM src
+GROUP BY key
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view15
+POSTHOOK: query: CREATE VIEW view15 AS
+SELECT key,COUNT(value) AS value_count
+FROM src
+GROUP BY key
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view15
+POSTHOOK: Lineage: view15.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: view15.value_count EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view15
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view15
+POSTHOOK: query: DESCRIBE EXTENDED view15
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view15
+key string
+value_count bigint
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view15
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view15
+POSTHOOK: query: DESCRIBE FORMATTED view15
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view15
+# col_name data_type comment
+key string
+value_count bigint
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT key,COUNT(value) AS value_count
+ FROM src
+ GROUP BY key
+Expanded Query: SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count`
+ FROM `default`.`src`
+ GROUP BY `src`.`key`
+PREHOOK: query: SELECT * FROM view15
+ORDER BY value_count DESC, key
+LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view15
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view15
+ORDER BY value_count DESC, key
+LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view15
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+230 5
+348 5
+401 5
+469 5
+138 4
+169 4
+277 4
+406 4
+468 4
+489 4
+PREHOOK: query: CREATE VIEW view16 AS
+SELECT DISTINCT value
+FROM src
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view16
+POSTHOOK: query: CREATE VIEW view16 AS
+SELECT DISTINCT value
+FROM src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view16
+POSTHOOK: Lineage: view16.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE EXTENDED view16
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view16
+POSTHOOK: query: DESCRIBE EXTENDED view16
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view16
+value string
+
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED view16
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view16
+POSTHOOK: query: DESCRIBE FORMATTED view16
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view16
+# col_name data_type comment
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+ bucketing_version 2
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+Original Query: SELECT DISTINCT value
+ FROM src
+Expanded Query: SELECT DISTINCT `src`.`value`
+ FROM `default`.`src`
+PREHOOK: query: SELECT * FROM view16
+ORDER BY value
+LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@view16
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM view16
+ORDER BY value
+LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@view16
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+val_0
+val_10
+val_100
+val_103
+val_104
+val_105
+val_11
+val_111
+val_113
+val_114
+PREHOOK: query: DROP TABLE IF EXISTS view16
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@view16
+PREHOOK: Output: default@view16
+POSTHOOK: query: DROP TABLE IF EXISTS view16
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@view16
+POSTHOOK: Output: default@view16
+PREHOOK: query: DESCRIBE view16
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@view16
+POSTHOOK: query: DESCRIBE view16
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@view16
+value string
+PREHOOK: query: DROP VIEW IF EXISTS table1_n4
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@table1_n4
+PREHOOK: Output: default@table1_n4
+POSTHOOK: query: DROP VIEW IF EXISTS table1_n4
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@table1_n4
+POSTHOOK: Output: default@table1_n4
+PREHOOK: query: DESCRIBE table1_n4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@table1_n4
+POSTHOOK: query: DESCRIBE table1_n4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@table1_n4
+key int
+value string
+PREHOOK: query: create view view17 as select 1 as v
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view17
+POSTHOOK: query: create view view17 as select 1 as v
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view17
+POSTHOOK: Lineage: view17.v SIMPLE []
+PREHOOK: query: select * from view17
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Input: default@view17
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from view17
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Input: default@view17
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: create view view18 as select v+1 from (select 1 as v) t
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view18
+POSTHOOK: query: create view view18 as select v+1 from (select 1 as v) t
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view18
+POSTHOOK: Lineage: view18._c0 SIMPLE []
+PREHOOK: query: select * from view18
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Input: default@view18
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from view18
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Input: default@view18
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: DROP VIEW view1
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view1
+PREHOOK: Output: default@view1
+POSTHOOK: query: DROP VIEW view1
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view1
+POSTHOOK: Output: default@view1
+PREHOOK: query: DROP VIEW view2
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view2
+PREHOOK: Output: default@view2
+POSTHOOK: query: DROP VIEW view2
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view2
+POSTHOOK: Output: default@view2
+PREHOOK: query: DROP VIEW view3
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view3
+PREHOOK: Output: default@view3
+POSTHOOK: query: DROP VIEW view3
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view3
+POSTHOOK: Output: default@view3
+PREHOOK: query: DROP VIEW view4
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view4
+PREHOOK: Output: default@view4
+POSTHOOK: query: DROP VIEW view4
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view4
+POSTHOOK: Output: default@view4
+PREHOOK: query: DROP VIEW view5
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view5
+PREHOOK: Output: default@view5
+POSTHOOK: query: DROP VIEW view5
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view5
+POSTHOOK: Output: default@view5
+PREHOOK: query: DROP VIEW view6
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view6
+PREHOOK: Output: default@view6
+POSTHOOK: query: DROP VIEW view6
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view6
+POSTHOOK: Output: default@view6
+PREHOOK: query: DROP VIEW view7
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view7
+PREHOOK: Output: default@view7
+POSTHOOK: query: DROP VIEW view7
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view7
+POSTHOOK: Output: default@view7
+PREHOOK: query: DROP VIEW view8
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view8
+PREHOOK: Output: default@view8
+POSTHOOK: query: DROP VIEW view8
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view8
+POSTHOOK: Output: default@view8
+PREHOOK: query: DROP VIEW view9
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view9
+PREHOOK: Output: default@view9
+POSTHOOK: query: DROP VIEW view9
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view9
+POSTHOOK: Output: default@view9
+PREHOOK: query: DROP VIEW view10
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view10
+PREHOOK: Output: default@view10
+POSTHOOK: query: DROP VIEW view10
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view10
+POSTHOOK: Output: default@view10
+PREHOOK: query: DROP VIEW view11
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view11
+PREHOOK: Output: default@view11
+POSTHOOK: query: DROP VIEW view11
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view11
+POSTHOOK: Output: default@view11
+PREHOOK: query: DROP VIEW view12
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view12
+PREHOOK: Output: default@view12
+POSTHOOK: query: DROP VIEW view12
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view12
+POSTHOOK: Output: default@view12
+PREHOOK: query: DROP VIEW view13
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view13
+PREHOOK: Output: default@view13
+POSTHOOK: query: DROP VIEW view13
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view13
+POSTHOOK: Output: default@view13
+PREHOOK: query: DROP VIEW view14
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view14
+PREHOOK: Output: default@view14
+POSTHOOK: query: DROP VIEW view14
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view14
+POSTHOOK: Output: default@view14
+PREHOOK: query: DROP VIEW view15
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view15
+PREHOOK: Output: default@view15
+POSTHOOK: query: DROP VIEW view15
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view15
+POSTHOOK: Output: default@view15
+PREHOOK: query: DROP VIEW view16
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view16
+PREHOOK: Output: default@view16
+POSTHOOK: query: DROP VIEW view16
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view16
+POSTHOOK: Output: default@view16
+PREHOOK: query: DROP VIEW view17
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view17
+PREHOOK: Output: default@view17
+POSTHOOK: query: DROP VIEW view17
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view17
+POSTHOOK: Output: default@view17
+PREHOOK: query: DROP VIEW view18
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@view18
+PREHOOK: Output: default@view18
+POSTHOOK: query: DROP VIEW view18
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@view18
+POSTHOOK: Output: default@view18
+PREHOOK: query: DROP TEMPORARY FUNCTION test_translate
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: test_translate
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_translate
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: test_translate
+PREHOOK: query: DROP TEMPORARY FUNCTION test_max
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: test_max
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_max
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: test_max
+PREHOOK: query: DROP TEMPORARY FUNCTION test_explode
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: test_explode
+POSTHOOK: query: DROP TEMPORARY FUNCTION test_explode
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: test_explode
diff --git a/ql/src/test/results/clientpositive/llap/skip_header_footer_aggr.q.out b/ql/src/test/results/clientpositive/llap/skip_header_footer_aggr.q.out
new file mode 100644
index 0000000..530e81f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/skip_header_footer_aggr.q.out
@@ -0,0 +1,849 @@
+PREHOOK: query: CREATE TABLE numbrs (numbr int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@numbrs
+POSTHOOK: query: CREATE TABLE numbrs (numbr int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@numbrs
+PREHOOK: query: INSERT INTO numbrs VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12), (NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@numbrs
+POSTHOOK: query: INSERT INTO numbrs VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12), (NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@numbrs
+POSTHOOK: Lineage: numbrs.numbr SCRIPT []
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@header_footer_table_4
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@header_footer_table_4
+PREHOOK: query: SELECT * FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+12 alex daily
+3 barry yearly
+5 chelsea monthly
+8 xavier monthly
+9 derek yearly
+11 ethan monthly
+1 faith yearly
+21 yves daily
+PREHOOK: query: SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 faith yearly
+3 barry yearly
+5 chelsea monthly
+8 xavier monthly
+9 derek yearly
+11 ethan monthly
+12 alex daily
+21 yves daily
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+5 chelsea monthly
+8 xavier monthly
+11 ethan monthly
+PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+8
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice is NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice is NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+8.0
+16.5
+4.333333333333333
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 faith yearly 1 faith yearly
+1 faith yearly 3 barry yearly
+1 faith yearly 5 chelsea monthly
+1 faith yearly 8 xavier monthly
+1 faith yearly 9 derek yearly
+1 faith yearly 11 ethan monthly
+1 faith yearly 12 alex daily
+1 faith yearly 21 yves daily
+3 barry yearly 1 faith yearly
+3 barry yearly 3 barry yearly
+3 barry yearly 5 chelsea monthly
+3 barry yearly 8 xavier monthly
+3 barry yearly 9 derek yearly
+3 barry yearly 11 ethan monthly
+3 barry yearly 12 alex daily
+3 barry yearly 21 yves daily
+5 chelsea monthly 1 faith yearly
+5 chelsea monthly 3 barry yearly
+5 chelsea monthly 5 chelsea monthly
+5 chelsea monthly 8 xavier monthly
+5 chelsea monthly 9 derek yearly
+5 chelsea monthly 11 ethan monthly
+5 chelsea monthly 12 alex daily
+5 chelsea monthly 21 yves daily
+8 xavier monthly 1 faith yearly
+8 xavier monthly 3 barry yearly
+8 xavier monthly 5 chelsea monthly
+8 xavier monthly 8 xavier monthly
+8 xavier monthly 9 derek yearly
+8 xavier monthly 11 ethan monthly
+8 xavier monthly 12 alex daily
+8 xavier monthly 21 yves daily
+9 derek yearly 1 faith yearly
+9 derek yearly 3 barry yearly
+9 derek yearly 5 chelsea monthly
+9 derek yearly 8 xavier monthly
+9 derek yearly 9 derek yearly
+9 derek yearly 11 ethan monthly
+9 derek yearly 12 alex daily
+9 derek yearly 21 yves daily
+11 ethan monthly 1 faith yearly
+11 ethan monthly 3 barry yearly
+11 ethan monthly 5 chelsea monthly
+11 ethan monthly 8 xavier monthly
+11 ethan monthly 9 derek yearly
+11 ethan monthly 11 ethan monthly
+11 ethan monthly 12 alex daily
+11 ethan monthly 21 yves daily
+12 alex daily 1 faith yearly
+12 alex daily 3 barry yearly
+12 alex daily 5 chelsea monthly
+12 alex daily 8 xavier monthly
+12 alex daily 9 derek yearly
+12 alex daily 11 ethan monthly
+12 alex daily 12 alex daily
+12 alex daily 21 yves daily
+21 yves daily 1 faith yearly
+21 yves daily 3 barry yearly
+21 yves daily 5 chelsea monthly
+21 yves daily 8 xavier monthly
+21 yves daily 9 derek yearly
+21 yves daily 11 ethan monthly
+21 yves daily 12 alex daily
+21 yves daily 21 yves daily
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith 1
+barry 3
+chelsea 5
+xavier 8
+derek 9
+ethan 11
+alex 12
+yves 21
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+NULL NULL
+faith 1
+NULL NULL
+barry 3
+NULL NULL
+chelsea 5
+NULL NULL
+NULL NULL
+xavier 8
+derek 9
+NULL NULL
+ethan 11
+alex 12
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith 1
+barry 3
+chelsea 5
+xavier 8
+derek 9
+ethan 11
+alex 12
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+NULL NULL
+faith 1
+NULL NULL
+barry 3
+NULL NULL
+chelsea 5
+NULL NULL
+NULL NULL
+xavier 8
+derek 9
+NULL NULL
+ethan 11
+alex 12
+yves 21
+PREHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+monthly 24
+daily 33
+yearly 13
+PREHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+monthly 24
+daily 33
+yearly 13
+PREHOOK: query: SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith yearly
+barry yearly
+chelsea monthly
+xavier monthly
+derek yearly
+ethan monthly
+alex daily
+yves daily
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@transition
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@transition
+PREHOOK: query: INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: default@transition
+POSTHOOK: query: INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: default@transition
+POSTHOOK: Lineage: transition.name EXPRESSION [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_name, type:string, comment:null), ]
+POSTHOOK: Lineage: transition.title SIMPLE [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_choice, type:varchar(10), comment:null), ]
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@transition
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@transition
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+daily alex daily alex
+daily alex daily yves
+daily alex monthly chelsea
+daily alex monthly ethan
+daily alex monthly xavier
+daily alex yearly barry
+daily alex yearly derek
+daily alex yearly faith
+daily yves daily alex
+daily yves daily yves
+daily yves monthly chelsea
+daily yves monthly ethan
+daily yves monthly xavier
+daily yves yearly barry
+daily yves yearly derek
+daily yves yearly faith
+monthly chelsea daily alex
+monthly chelsea daily yves
+monthly chelsea monthly chelsea
+monthly chelsea monthly ethan
+monthly chelsea monthly xavier
+monthly chelsea yearly barry
+monthly chelsea yearly derek
+monthly chelsea yearly faith
+monthly ethan daily alex
+monthly ethan daily yves
+monthly ethan monthly chelsea
+monthly ethan monthly ethan
+monthly ethan monthly xavier
+monthly ethan yearly barry
+monthly ethan yearly derek
+monthly ethan yearly faith
+monthly xavier daily alex
+monthly xavier daily yves
+monthly xavier monthly chelsea
+monthly xavier monthly ethan
+monthly xavier monthly xavier
+monthly xavier yearly barry
+monthly xavier yearly derek
+monthly xavier yearly faith
+yearly barry daily alex
+yearly barry daily yves
+yearly barry monthly chelsea
+yearly barry monthly ethan
+yearly barry monthly xavier
+yearly barry yearly barry
+yearly barry yearly derek
+yearly barry yearly faith
+yearly derek daily alex
+yearly derek daily yves
+yearly derek monthly chelsea
+yearly derek monthly ethan
+yearly derek monthly xavier
+yearly derek yearly barry
+yearly derek yearly derek
+yearly derek yearly faith
+yearly faith daily alex
+yearly faith daily yves
+yearly faith monthly chelsea
+yearly faith monthly ethan
+yearly faith monthly xavier
+yearly faith yearly barry
+yearly faith yearly derek
+yearly faith yearly faith
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@transition2
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@transition2
+PREHOOK: query: INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: default@transition2
+POSTHOOK: query: INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: default@transition2
+POSTHOOK: Lineage: transition2.header_choice SIMPLE [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_choice, type:varchar(10), comment:null), ]
+POSTHOOK: Lineage: transition2.sum_header_int EXPRESSION [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_int, type:int, comment:null), ]
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@transition2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@transition2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+yearly 13 yearly 13
+yearly 13 monthly 24
+yearly 13 daily 33
+monthly 24 yearly 13
+monthly 24 monthly 24
+monthly 24 daily 33
+daily 33 yearly 13
+daily 33 monthly 24
+daily 33 daily 33
+PREHOOK: query: DROP TABLE transition
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@transition
+PREHOOK: Output: default@transition
+POSTHOOK: query: DROP TABLE transition
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@transition
+POSTHOOK: Output: default@transition
+PREHOOK: query: DROP TABLE transition2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@transition2
+PREHOOK: Output: default@transition2
+POSTHOOK: query: DROP TABLE transition2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@transition2
+POSTHOOK: Output: default@transition2
+PREHOOK: query: SELECT * FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+12 alex daily
+3 barry yearly
+5 chelsea monthly
+8 xavier monthly
+9 derek yearly
+11 ethan monthly
+1 faith yearly
+21 yves daily
+PREHOOK: query: SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 faith yearly
+3 barry yearly
+5 chelsea monthly
+8 xavier monthly
+9 derek yearly
+11 ethan monthly
+12 alex daily
+21 yves daily
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'header_choice'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice = 'monthly'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+5 chelsea monthly
+8 xavier monthly
+11 ethan monthly
+PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+8
+PREHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice is NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 WHERE header_choice is NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT AVG(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+8.0
+16.5
+4.333333333333333
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM header_footer_table_4 A, header_footer_table_4 B ORDER BY A.header_int, B.header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 faith yearly 1 faith yearly
+1 faith yearly 3 barry yearly
+1 faith yearly 5 chelsea monthly
+1 faith yearly 8 xavier monthly
+1 faith yearly 9 derek yearly
+1 faith yearly 11 ethan monthly
+1 faith yearly 12 alex daily
+1 faith yearly 21 yves daily
+3 barry yearly 1 faith yearly
+3 barry yearly 3 barry yearly
+3 barry yearly 5 chelsea monthly
+3 barry yearly 8 xavier monthly
+3 barry yearly 9 derek yearly
+3 barry yearly 11 ethan monthly
+3 barry yearly 12 alex daily
+3 barry yearly 21 yves daily
+5 chelsea monthly 1 faith yearly
+5 chelsea monthly 3 barry yearly
+5 chelsea monthly 5 chelsea monthly
+5 chelsea monthly 8 xavier monthly
+5 chelsea monthly 9 derek yearly
+5 chelsea monthly 11 ethan monthly
+5 chelsea monthly 12 alex daily
+5 chelsea monthly 21 yves daily
+8 xavier monthly 1 faith yearly
+8 xavier monthly 3 barry yearly
+8 xavier monthly 5 chelsea monthly
+8 xavier monthly 8 xavier monthly
+8 xavier monthly 9 derek yearly
+8 xavier monthly 11 ethan monthly
+8 xavier monthly 12 alex daily
+8 xavier monthly 21 yves daily
+9 derek yearly 1 faith yearly
+9 derek yearly 3 barry yearly
+9 derek yearly 5 chelsea monthly
+9 derek yearly 8 xavier monthly
+9 derek yearly 9 derek yearly
+9 derek yearly 11 ethan monthly
+9 derek yearly 12 alex daily
+9 derek yearly 21 yves daily
+11 ethan monthly 1 faith yearly
+11 ethan monthly 3 barry yearly
+11 ethan monthly 5 chelsea monthly
+11 ethan monthly 8 xavier monthly
+11 ethan monthly 9 derek yearly
+11 ethan monthly 11 ethan monthly
+11 ethan monthly 12 alex daily
+11 ethan monthly 21 yves daily
+12 alex daily 1 faith yearly
+12 alex daily 3 barry yearly
+12 alex daily 5 chelsea monthly
+12 alex daily 8 xavier monthly
+12 alex daily 9 derek yearly
+12 alex daily 11 ethan monthly
+12 alex daily 12 alex daily
+12 alex daily 21 yves daily
+21 yves daily 1 faith yearly
+21 yves daily 3 barry yearly
+21 yves daily 5 chelsea monthly
+21 yves daily 8 xavier monthly
+21 yves daily 9 derek yearly
+21 yves daily 11 ethan monthly
+21 yves daily 12 alex daily
+21 yves daily 21 yves daily
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 LEFT JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith 1
+barry 3
+chelsea 5
+xavier 8
+derek 9
+ethan 11
+alex 12
+yves 21
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 RIGHT JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+NULL NULL
+faith 1
+NULL NULL
+barry 3
+NULL NULL
+chelsea 5
+NULL NULL
+NULL NULL
+xavier 8
+derek 9
+NULL NULL
+ethan 11
+alex 12
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 INNER JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith 1
+barry 3
+chelsea 5
+xavier 8
+derek 9
+ethan 11
+alex 12
+PREHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Input: default@numbrs
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_int FROM header_footer_table_4 FULL JOIN numbrs ON numbr = header_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Input: default@numbrs
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+NULL NULL
+faith 1
+NULL NULL
+barry 3
+NULL NULL
+chelsea 5
+NULL NULL
+NULL NULL
+xavier 8
+derek 9
+NULL NULL
+ethan 11
+alex 12
+yves 21
+PREHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+monthly 24
+daily 33
+yearly 13
+PREHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+monthly 24
+daily 33
+yearly 13
+PREHOOK: query: SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT header_name, header_choice FROM header_footer_table_4 ORDER BY header_int LIMIT 8
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+faith yearly
+barry yearly
+chelsea monthly
+xavier monthly
+derek yearly
+ethan monthly
+alex daily
+yves daily
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@transition
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@transition
+PREHOOK: query: INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: default@transition
+POSTHOOK: query: INSERT INTO transition SELECT header_choice, header_name FROM header_footer_table_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: default@transition
+POSTHOOK: Lineage: transition.name EXPRESSION [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_name, type:string, comment:null), ]
+POSTHOOK: Lineage: transition.title SIMPLE [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_choice, type:varchar(10), comment:null), ]
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@transition
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM transition A, transition B ORDER BY A.title, A.name, B.title, B.name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@transition
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+daily alex daily alex
+daily alex daily yves
+daily alex monthly chelsea
+daily alex monthly ethan
+daily alex monthly xavier
+daily alex yearly barry
+daily alex yearly derek
+daily alex yearly faith
+daily yves daily alex
+daily yves daily yves
+daily yves monthly chelsea
+daily yves monthly ethan
+daily yves monthly xavier
+daily yves yearly barry
+daily yves yearly derek
+daily yves yearly faith
+monthly chelsea daily alex
+monthly chelsea daily yves
+monthly chelsea monthly chelsea
+monthly chelsea monthly ethan
+monthly chelsea monthly xavier
+monthly chelsea yearly barry
+monthly chelsea yearly derek
+monthly chelsea yearly faith
+monthly ethan daily alex
+monthly ethan daily yves
+monthly ethan monthly chelsea
+monthly ethan monthly ethan
+monthly ethan monthly xavier
+monthly ethan yearly barry
+monthly ethan yearly derek
+monthly ethan yearly faith
+monthly xavier daily alex
+monthly xavier daily yves
+monthly xavier monthly chelsea
+monthly xavier monthly ethan
+monthly xavier monthly xavier
+monthly xavier yearly barry
+monthly xavier yearly derek
+monthly xavier yearly faith
+yearly barry daily alex
+yearly barry daily yves
+yearly barry monthly chelsea
+yearly barry monthly ethan
+yearly barry monthly xavier
+yearly barry yearly barry
+yearly barry yearly derek
+yearly barry yearly faith
+yearly derek daily alex
+yearly derek daily yves
+yearly derek monthly chelsea
+yearly derek monthly ethan
+yearly derek monthly xavier
+yearly derek yearly barry
+yearly derek yearly derek
+yearly derek yearly faith
+yearly faith daily alex
+yearly faith daily yves
+yearly faith monthly chelsea
+yearly faith monthly ethan
+yearly faith monthly xavier
+yearly faith yearly barry
+yearly faith yearly derek
+yearly faith yearly faith
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@transition2
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@transition2
+PREHOOK: query: INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_4
+PREHOOK: Output: default@transition2
+POSTHOOK: query: INSERT INTO transition2 SELECT header_choice, SUM(header_int) FROM header_footer_table_4 GROUP BY header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_4
+POSTHOOK: Output: default@transition2
+POSTHOOK: Lineage: transition2.header_choice SIMPLE [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_choice, type:varchar(10), comment:null), ]
+POSTHOOK: Lineage: transition2.sum_header_int EXPRESSION [(header_footer_table_4)header_footer_table_4.FieldSchema(name:header_int, type:int, comment:null), ]
+Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+PREHOOK: query: SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@transition2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM transition2 A, transition2 B ORDER BY A.sum_header_int, A.header_choice, B.sum_header_int, B.header_choice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@transition2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+yearly 13 yearly 13
+yearly 13 monthly 24
+yearly 13 daily 33
+monthly 24 yearly 13
+monthly 24 monthly 24
+monthly 24 daily 33
+daily 33 yearly 13
+daily 33 monthly 24
+daily 33 daily 33
diff --git a/ql/src/test/results/clientpositive/llap/skip_header_footer_proj.q.out b/ql/src/test/results/clientpositive/llap/skip_header_footer_proj.q.out
new file mode 100644
index 0000000..f78f7fb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/skip_header_footer_proj.q.out
@@ -0,0 +1,123 @@
+PREHOOK: query: DROP TABLE IF EXISTS hf1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hf1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hf1 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hf1
+POSTHOOK: query: CREATE TABLE hf1 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hf1
+PREHOOK: query: INSERT OVERWRITE TABLE hf1 VALUES ('x','y'),('a','b'),('c','d')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hf1
+POSTHOOK: query: INSERT OVERWRITE TABLE hf1 VALUES ('x','y'),('a','b'),('c','d')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hf1
+POSTHOOK: Lineage: hf1.a SCRIPT []
+POSTHOOK: Lineage: hf1.b SCRIPT []
+PREHOOK: query: SELECT * FROM hf1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hf1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM hf1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hf1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+a b
+c d
+PREHOOK: query: DROP TABLE IF EXISTS hf2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hf2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hf2 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.footer.line.count'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hf2
+POSTHOOK: query: CREATE TABLE hf2 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.footer.line.count'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hf2
+PREHOOK: query: INSERT OVERWRITE TABLE hf2 VALUES ('x','y'),('a','b'),('c','d')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hf2
+POSTHOOK: query: INSERT OVERWRITE TABLE hf2 VALUES ('x','y'),('a','b'),('c','d')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hf2
+POSTHOOK: Lineage: hf2.a SCRIPT []
+POSTHOOK: Lineage: hf2.b SCRIPT []
+PREHOOK: query: SELECT * FROM hf2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hf2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM hf2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hf2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+x y
+PREHOOK: query: DROP TABLE IF EXISTS hf3
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hf3
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hf3 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hf3
+POSTHOOK: query: CREATE TABLE hf3 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hf3
+PREHOOK: query: INSERT OVERWRITE TABLE hf3 VALUES ('x','y'),('a','b'),('c','d')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hf3
+POSTHOOK: query: INSERT OVERWRITE TABLE hf3 VALUES ('x','y'),('a','b'),('c','d')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hf3
+POSTHOOK: Lineage: hf3.a SCRIPT []
+POSTHOOK: Lineage: hf3.b SCRIPT []
+PREHOOK: query: SELECT * FROM hf3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hf3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM hf3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hf3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: DROP TABLE IF EXISTS hf4
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hf4
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hf4 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1','skip.footer.line.count'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hf4
+POSTHOOK: query: CREATE TABLE hf4 (a string, b string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' TBLPROPERTIES('skip.header.line.count'='1','skip.footer.line.count'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hf4
+PREHOOK: query: INSERT OVERWRITE TABLE hf4 VALUES ('x','y'),('a','b'),('c','d')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hf4
+POSTHOOK: query: INSERT OVERWRITE TABLE hf4 VALUES ('x','y'),('a','b'),('c','d')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hf4
+POSTHOOK: Lineage: hf4.a SCRIPT []
+POSTHOOK: Lineage: hf4.b SCRIPT []
+PREHOOK: query: SELECT * FROM hf4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hf4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM hf4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hf4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
diff --git a/ql/src/test/results/clientpositive/tez/compressed_skip_header_footer_aggr.q.out b/ql/src/test/results/clientpositive/tez/compressed_skip_header_footer_aggr.q.out
new file mode 100644
index 0000000..a42e75e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/compressed_skip_header_footer_aggr.q.out
@@ -0,0 +1,362 @@
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase1`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase1
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase1`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase1
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase2`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase2
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase2`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase2
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase3`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase3
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase3`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase3
+PREHOOK: query: select * from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+3 2019-12-31
+PREHOOK: query: select count(*) from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31
+3 2019-12-31
+PREHOOK: query: select count(*) from testcase2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31
+2 2019-12-31
+PREHOOK: query: select count(*) from testcase3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase3
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase3
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase4`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase4
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase4`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase4
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase5`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase5
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase5`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase5
+PREHOOK: query: CREATE EXTERNAL TABLE `testcase6`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase6
+POSTHOOK: query: CREATE EXTERNAL TABLE `testcase6`(id int, name string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+#### A masked pattern was here ####
+ TBLPROPERTIES ("skip.header.line.count"="0", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase6
+PREHOOK: query: select * from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+3 2019-12-31 02
+PREHOOK: query: select count(*) from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31 00
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase4
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase4
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+PREHOOK: query: select * from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2 2019-12-31 01
+3 2019-12-31 02
+PREHOOK: query: select count(*) from testcase5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase5
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase5
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 2019-12-31 00
+2 2019-12-31 01
+PREHOOK: query: select count(*) from testcase6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase6
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase6
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: create table testcase_gz(age int, name string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testcase_gz
+POSTHOOK: query: create table testcase_gz(age int, name string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+TBLPROPERTIES ("skip.header.line.count"="1", "skip.footer.line.count"="1")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testcase_gz
+PREHOOK: query: select * from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a
+2 b
+PREHOOK: query: select count(*) from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
+PREHOOK: query: select * from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 a
+2 b
+PREHOOK: query: select count(*) from testcase_gz
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testcase_gz
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from testcase_gz
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testcase_gz
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2
