| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.io.compress.zlib; |
| |
| import java.io.IOException; |
| import java.util.zip.DataFormatException; |
| import java.util.zip.Inflater; |
| |
| import org.apache.hadoop.util.PureJavaCrc32; |
| import org.apache.hadoop.io.compress.Decompressor; |
| import org.apache.hadoop.io.compress.DoNotPool; |
| |
| /** |
| * A {@link Decompressor} based on the popular gzip compressed file format. |
| * http://www.gzip.org/ |
| * |
| */ |
| @DoNotPool |
| public class BuiltInGzipDecompressor implements Decompressor { |
| private static final int GZIP_MAGIC_ID = 0x8b1f; // if read as LE short int |
| private static final int GZIP_DEFLATE_METHOD = 8; |
| private static final int GZIP_FLAGBIT_HEADER_CRC = 0x02; |
| private static final int GZIP_FLAGBIT_EXTRA_FIELD = 0x04; |
| private static final int GZIP_FLAGBIT_FILENAME = 0x08; |
| private static final int GZIP_FLAGBIT_COMMENT = 0x10; |
| private static final int GZIP_FLAGBITS_RESERVED = 0xe0; |
| |
| // 'true' (nowrap) => Inflater will handle raw deflate stream only |
| private Inflater inflater = new Inflater(true); |
| |
| private byte[] userBuf = null; |
| private int userBufOff = 0; |
| private int userBufLen = 0; |
| |
| private byte[] localBuf = new byte[256]; |
| private int localBufOff = 0; |
| |
| private int headerBytesRead = 0; |
| private int trailerBytesRead = 0; |
| private int numExtraFieldBytesRemaining = -1; |
| private PureJavaCrc32 crc = new PureJavaCrc32(); |
| private boolean hasExtraField = false; |
| private boolean hasFilename = false; |
| private boolean hasComment = false; |
| private boolean hasHeaderCRC = false; |
| |
| private GzipStateLabel state; |
| |
| /** |
| * The current state of the gzip decoder, external to the Inflater context. |
| * (Technically, the private variables localBuf through hasHeaderCRC are |
| * also part of the state, so this enum is merely the label for it.) |
| */ |
| private static enum GzipStateLabel { |
| /** |
| * Immediately prior to or (strictly) within the 10-byte basic gzip header. |
| */ |
| HEADER_BASIC, |
| /** |
| * Immediately prior to or within the optional "extra field." |
| */ |
| HEADER_EXTRA_FIELD, |
| /** |
| * Immediately prior to or within the optional filename field. |
| */ |
| HEADER_FILENAME, |
| /** |
| * Immediately prior to or within the optional comment field. |
| */ |
| HEADER_COMMENT, |
| /** |
| * Immediately prior to or within the optional 2-byte header CRC value. |
| */ |
| HEADER_CRC, |
| /** |
| * Immediately prior to or within the main compressed (deflate) data stream. |
| */ |
| DEFLATE_STREAM, |
| /** |
| * Immediately prior to or (strictly) within the 4-byte uncompressed CRC. |
| */ |
| TRAILER_CRC, |
| /** |
| * Immediately prior to or (strictly) within the 4-byte uncompressed size. |
| */ |
| TRAILER_SIZE, |
| /** |
| * Immediately after the trailer (and potentially prior to the next gzip |
| * member/substream header), without reset() having been called. |
| */ |
| FINISHED; |
| } |
| |
| /** |
| * Creates a new (pure Java) gzip decompressor. |
| */ |
| public BuiltInGzipDecompressor() { |
| state = GzipStateLabel.HEADER_BASIC; |
| crc.reset(); |
| // FIXME? Inflater docs say: 'it is also necessary to provide an extra |
| // "dummy" byte as input. This is required by the ZLIB native |
| // library in order to support certain optimizations.' However, |
| // this does not appear to be true, and in any case, it's not |
| // entirely clear where the byte should go or what its value |
| // should be. Perhaps it suffices to have some deflated bytes |
| // in the first buffer load? (But how else would one do it?) |
| } |
| |
| /** {@inheritDoc} */ |
| public synchronized boolean needsInput() { |
| if (state == GzipStateLabel.DEFLATE_STREAM) { // most common case |
| return inflater.needsInput(); |
| } |
| // see userBufLen comment at top of decompress(); currently no need to |
| // verify userBufLen <= 0 |
| return (state != GzipStateLabel.FINISHED); |
| } |
| |
| /** {@inheritDoc} */ |
| /* |
| * In our case, the input data includes both gzip header/trailer bytes (which |
| * we handle in executeState()) and deflate-stream bytes (which we hand off |
| * to Inflater). |
| * |
| * NOTE: This code assumes the data passed in via b[] remains unmodified |
| * until _we_ signal that it's safe to modify it (via needsInput()). |
| * The alternative would require an additional buffer-copy even for |
| * the bulk deflate stream, which is a performance hit we don't want |
| * to absorb. (Decompressor now documents this requirement.) |
| */ |
| public synchronized void setInput(byte[] b, int off, int len) { |
| if (b == null) { |
| throw new NullPointerException(); |
| } |
| if (off < 0 || len < 0 || off > b.length - len) { |
| throw new ArrayIndexOutOfBoundsException(); |
| } |
| |
| userBuf = b; |
| userBufOff = off; |
| userBufLen = len; // note: might be zero |
| } |
| |
| /** |
| * Decompress the data (gzip header, deflate stream, gzip trailer) in the |
| * provided buffer. |
| * |
| * @return the number of decompressed bytes placed into b |
| */ |
| /* From the caller's perspective, this is where the state machine lives. |
| * The code is written such that we never return from decompress() with |
| * data remaining in userBuf unless we're in FINISHED state and there was |
| * data beyond the current gzip member (e.g., we're within a concatenated |
| * gzip stream). If this ever changes, {@link #needsInput()} will also |
| * need to be modified (i.e., uncomment the userBufLen condition). |
| * |
| * The actual deflate-stream processing (decompression) is handled by |
| * Java's Inflater class. Unlike the gzip header/trailer code (execute* |
| * methods below), the deflate stream is never copied; Inflater operates |
| * directly on the user's buffer. |
| */ |
| public synchronized int decompress(byte[] b, int off, int len) |
| throws IOException { |
| int numAvailBytes = 0; |
| |
| if (state != GzipStateLabel.DEFLATE_STREAM) { |
| executeHeaderState(); |
| |
| if (userBufLen <= 0) { |
| return numAvailBytes; |
| } |
| } |
| |
| // "executeDeflateStreamState()" |
| if (state == GzipStateLabel.DEFLATE_STREAM) { |
| // hand off user data (or what's left of it) to Inflater--but note that |
| // Inflater may not have consumed all of previous bufferload (e.g., if |
| // data highly compressed or output buffer very small), in which case |
| // userBufLen will be zero |
| if (userBufLen > 0) { |
| inflater.setInput(userBuf, userBufOff, userBufLen); |
| userBufOff += userBufLen; |
| userBufLen = 0; |
| } |
| |
| // now decompress it into b[] |
| try { |
| numAvailBytes = inflater.inflate(b, off, len); |
| } catch (DataFormatException dfe) { |
| throw new IOException(dfe.getMessage()); |
| } |
| crc.update(b, off, numAvailBytes); // CRC-32 is on _uncompressed_ data |
| if (inflater.finished()) { |
| state = GzipStateLabel.TRAILER_CRC; |
| int bytesRemaining = inflater.getRemaining(); |
| assert (bytesRemaining >= 0) : |
| "logic error: Inflater finished; byte-count is inconsistent"; |
| // could save a copy of userBufLen at call to inflater.setInput() and |
| // verify that bytesRemaining <= origUserBufLen, but would have to |
| // be a (class) member variable...seems excessive for a sanity check |
| userBufOff -= bytesRemaining; |
| userBufLen = bytesRemaining; // or "+=", but guaranteed 0 coming in |
| } else { |
| return numAvailBytes; // minor optimization |
| } |
| } |
| |
| executeTrailerState(); |
| |
| return numAvailBytes; |
| } |
| |
| /** |
| * Parse the gzip header (assuming we're in the appropriate state). |
| * In order to deal with degenerate cases (e.g., user buffer is one byte |
| * long), we copy (some) header bytes to another buffer. (Filename, |
| * comment, and extra-field bytes are simply skipped.)</p> |
| * |
| * See http://www.ietf.org/rfc/rfc1952.txt for the gzip spec. Note that |
| * no version of gzip to date (at least through 1.4.0, 2010-01-20) supports |
| * the FHCRC header-CRC16 flagbit; instead, the implementation treats it |
| * as a multi-file continuation flag (which it also doesn't support). :-( |
| * Sun's JDK v6 (1.6) supports the header CRC, however, and so do we. |
| */ |
| private void executeHeaderState() throws IOException { |
| |
| // this can happen because DecompressorStream's decompress() is written |
| // to call decompress() first, setInput() second: |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| // "basic"/required header: somewhere in first 10 bytes |
| if (state == GzipStateLabel.HEADER_BASIC) { |
| int n = Math.min(userBufLen, 10-localBufOff); // (or 10-headerBytesRead) |
| checkAndCopyBytesToLocal(n); // modifies userBufLen, etc. |
| if (localBufOff >= 10) { // should be strictly == |
| processBasicHeader(); // sig, compression method, flagbits |
| localBufOff = 0; // no further need for basic header |
| state = GzipStateLabel.HEADER_EXTRA_FIELD; |
| } |
| } |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| // optional header stuff (extra field, filename, comment, header CRC) |
| |
| if (state == GzipStateLabel.HEADER_EXTRA_FIELD) { |
| if (hasExtraField) { |
| // 2 substates: waiting for 2 bytes => get numExtraFieldBytesRemaining, |
| // or already have 2 bytes & waiting to finish skipping specified length |
| if (numExtraFieldBytesRemaining < 0) { |
| int n = Math.min(userBufLen, 2-localBufOff); |
| checkAndCopyBytesToLocal(n); |
| if (localBufOff >= 2) { |
| numExtraFieldBytesRemaining = readUShortLE(localBuf, 0); |
| localBufOff = 0; |
| } |
| } |
| if (numExtraFieldBytesRemaining > 0 && userBufLen > 0) { |
| int n = Math.min(userBufLen, numExtraFieldBytesRemaining); |
| checkAndSkipBytes(n); // modifies userBufLen, etc. |
| numExtraFieldBytesRemaining -= n; |
| } |
| if (numExtraFieldBytesRemaining == 0) { |
| state = GzipStateLabel.HEADER_FILENAME; |
| } |
| } else { |
| state = GzipStateLabel.HEADER_FILENAME; |
| } |
| } |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| if (state == GzipStateLabel.HEADER_FILENAME) { |
| if (hasFilename) { |
| boolean doneWithFilename = checkAndSkipBytesUntilNull(); |
| if (!doneWithFilename) { |
| return; // exit early: used up entire buffer without hitting NULL |
| } |
| } |
| state = GzipStateLabel.HEADER_COMMENT; |
| } |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| if (state == GzipStateLabel.HEADER_COMMENT) { |
| if (hasComment) { |
| boolean doneWithComment = checkAndSkipBytesUntilNull(); |
| if (!doneWithComment) { |
| return; // exit early: used up entire buffer |
| } |
| } |
| state = GzipStateLabel.HEADER_CRC; |
| } |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| if (state == GzipStateLabel.HEADER_CRC) { |
| if (hasHeaderCRC) { |
| assert (localBufOff < 2); |
| int n = Math.min(userBufLen, 2-localBufOff); |
| copyBytesToLocal(n); |
| if (localBufOff >= 2) { |
| long headerCRC = readUShortLE(localBuf, 0); |
| if (headerCRC != (crc.getValue() & 0xffff)) { |
| throw new IOException("gzip header CRC failure"); |
| } |
| localBufOff = 0; |
| crc.reset(); |
| state = GzipStateLabel.DEFLATE_STREAM; |
| } |
| } else { |
| crc.reset(); // will reuse for CRC-32 of uncompressed data |
| state = GzipStateLabel.DEFLATE_STREAM; // switching to Inflater now |
| } |
| } |
| } |
| |
| /** |
| * Parse the gzip trailer (assuming we're in the appropriate state). |
| * In order to deal with degenerate cases (e.g., user buffer is one byte |
| * long), we copy trailer bytes (all 8 of 'em) to a local buffer.</p> |
| * |
| * See http://www.ietf.org/rfc/rfc1952.txt for the gzip spec. |
| */ |
| private void executeTrailerState() throws IOException { |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| // verify that the CRC-32 of the decompressed stream matches the value |
| // stored in the gzip trailer |
| if (state == GzipStateLabel.TRAILER_CRC) { |
| // localBuf was empty before we handed off to Inflater, so we handle this |
| // exactly like header fields |
| assert (localBufOff < 4); // initially 0, but may need multiple calls |
| int n = Math.min(userBufLen, 4-localBufOff); |
| copyBytesToLocal(n); |
| if (localBufOff >= 4) { |
| long streamCRC = readUIntLE(localBuf, 0); |
| if (streamCRC != crc.getValue()) { |
| throw new IOException("gzip stream CRC failure"); |
| } |
| localBufOff = 0; |
| crc.reset(); |
| state = GzipStateLabel.TRAILER_SIZE; |
| } |
| } |
| |
| if (userBufLen <= 0) { |
| return; |
| } |
| |
| // verify that the mod-2^32 decompressed stream size matches the value |
| // stored in the gzip trailer |
| if (state == GzipStateLabel.TRAILER_SIZE) { |
| assert (localBufOff < 4); // initially 0, but may need multiple calls |
| int n = Math.min(userBufLen, 4-localBufOff); |
| copyBytesToLocal(n); // modifies userBufLen, etc. |
| if (localBufOff >= 4) { // should be strictly == |
| long inputSize = readUIntLE(localBuf, 0); |
| if (inputSize != (inflater.getBytesWritten() & 0xffffffff)) { |
| throw new IOException( |
| "stored gzip size doesn't match decompressed size"); |
| } |
| localBufOff = 0; |
| state = GzipStateLabel.FINISHED; |
| } |
| } |
| |
| if (state == GzipStateLabel.FINISHED) { |
| return; |
| } |
| } |
| |
| /** |
| * Returns the total number of compressed bytes input so far, including |
| * gzip header/trailer bytes.</p> |
| * |
| * @return the total (non-negative) number of compressed bytes read so far |
| */ |
| public synchronized long getBytesRead() { |
| return headerBytesRead + inflater.getBytesRead() + trailerBytesRead; |
| } |
| |
| /** |
| * Returns the number of bytes remaining in the input buffer; normally |
| * called when finished() is true to determine amount of post-gzip-stream |
| * data. Note that, other than the finished state with concatenated data |
| * after the end of the current gzip stream, this will never return a |
| * non-zero value unless called after {@link #setInput(byte[] b, int off, |
| * int len)} and before {@link #decompress(byte[] b, int off, int len)}. |
| * (That is, after {@link #decompress(byte[] b, int off, int len)} it |
| * always returns zero, except in finished state with concatenated data.)</p> |
| * |
| * @return the total (non-negative) number of unprocessed bytes in input |
| */ |
| public synchronized int getRemaining() { |
| return userBufLen; |
| } |
| |
| /** {@inheritDoc} */ |
| public synchronized boolean needsDictionary() { |
| return inflater.needsDictionary(); |
| } |
| |
| /** {@inheritDoc} */ |
| public synchronized void setDictionary(byte[] b, int off, int len) { |
| inflater.setDictionary(b, off, len); |
| } |
| |
| /** |
| * Returns true if the end of the gzip substream (single "member") has been |
| * reached.</p> |
| */ |
| public synchronized boolean finished() { |
| return (state == GzipStateLabel.FINISHED); |
| } |
| |
| /** |
| * Resets everything, including the input buffer, regardless of whether the |
| * current gzip substream is finished.</p> |
| */ |
| public synchronized void reset() { |
| // could optionally emit INFO message if state != GzipStateLabel.FINISHED |
| inflater.reset(); |
| state = GzipStateLabel.HEADER_BASIC; |
| crc.reset(); |
| userBufOff = userBufLen = 0; |
| localBufOff = 0; |
| headerBytesRead = 0; |
| trailerBytesRead = 0; |
| numExtraFieldBytesRemaining = -1; |
| hasExtraField = false; |
| hasFilename = false; |
| hasComment = false; |
| hasHeaderCRC = false; |
| } |
| |
| /** {@inheritDoc} */ |
| public synchronized void end() { |
| inflater.end(); |
| } |
| |
| /** |
| * Check ID bytes (throw if necessary), compression method (throw if not 8), |
| * and flag bits (set hasExtraField, hasFilename, hasComment, hasHeaderCRC). |
| * Ignore MTIME, XFL, OS. Caller must ensure we have at least 10 bytes (at |
| * the start of localBuf).</p> |
| */ |
| /* |
| * Flag bits (remainder are reserved and must be zero): |
| * bit 0 FTEXT |
| * bit 1 FHCRC (never implemented in gzip, at least through version |
| * 1.4.0; instead interpreted as "continuation of multi- |
| * part gzip file," which is unsupported through 1.4.0) |
| * bit 2 FEXTRA |
| * bit 3 FNAME |
| * bit 4 FCOMMENT |
| * [bit 5 encrypted] |
| */ |
| private void processBasicHeader() throws IOException { |
| if (readUShortLE(localBuf, 0) != GZIP_MAGIC_ID) { |
| throw new IOException("not a gzip file"); |
| } |
| if (readUByte(localBuf, 2) != GZIP_DEFLATE_METHOD) { |
| throw new IOException("gzip data not compressed with deflate method"); |
| } |
| int flg = readUByte(localBuf, 3); |
| if ((flg & GZIP_FLAGBITS_RESERVED) != 0) { |
| throw new IOException("unknown gzip format (reserved flagbits set)"); |
| } |
| hasExtraField = ((flg & GZIP_FLAGBIT_EXTRA_FIELD) != 0); |
| hasFilename = ((flg & GZIP_FLAGBIT_FILENAME) != 0); |
| hasComment = ((flg & GZIP_FLAGBIT_COMMENT) != 0); |
| hasHeaderCRC = ((flg & GZIP_FLAGBIT_HEADER_CRC) != 0); |
| } |
| |
| private void checkAndCopyBytesToLocal(int len) { |
| System.arraycopy(userBuf, userBufOff, localBuf, localBufOff, len); |
| localBufOff += len; |
| // alternatively, could call checkAndSkipBytes(len) for rest... |
| crc.update(userBuf, userBufOff, len); |
| userBufOff += len; |
| userBufLen -= len; |
| headerBytesRead += len; |
| } |
| |
| private void checkAndSkipBytes(int len) { |
| crc.update(userBuf, userBufOff, len); |
| userBufOff += len; |
| userBufLen -= len; |
| headerBytesRead += len; |
| } |
| |
| // returns true if saw NULL, false if ran out of buffer first; called _only_ |
| // during gzip-header processing (not trailer) |
| // (caller can check before/after state of userBufLen to compute num bytes) |
| private boolean checkAndSkipBytesUntilNull() { |
| boolean hitNull = false; |
| if (userBufLen > 0) { |
| do { |
| hitNull = (userBuf[userBufOff] == 0); |
| crc.update(userBuf[userBufOff]); |
| ++userBufOff; |
| --userBufLen; |
| ++headerBytesRead; |
| } while (userBufLen > 0 && !hitNull); |
| } |
| return hitNull; |
| } |
| |
| // this one doesn't update the CRC and does support trailer processing but |
| // otherwise is same as its "checkAnd" sibling |
| private void copyBytesToLocal(int len) { |
| System.arraycopy(userBuf, userBufOff, localBuf, localBufOff, len); |
| localBufOff += len; |
| userBufOff += len; |
| userBufLen -= len; |
| if (state == GzipStateLabel.TRAILER_CRC || |
| state == GzipStateLabel.TRAILER_SIZE) { |
| trailerBytesRead += len; |
| } else { |
| headerBytesRead += len; |
| } |
| } |
| |
| private int readUByte(byte[] b, int off) { |
| return ((int)b[off] & 0xff); |
| } |
| |
| // caller is responsible for not overrunning buffer |
| private int readUShortLE(byte[] b, int off) { |
| return ((((b[off+1] & 0xff) << 8) | |
| ((b[off] & 0xff) )) & 0xffff); |
| } |
| |
| // caller is responsible for not overrunning buffer |
| private long readUIntLE(byte[] b, int off) { |
| return ((((long)(b[off+3] & 0xff) << 24) | |
| ((long)(b[off+2] & 0xff) << 16) | |
| ((long)(b[off+1] & 0xff) << 8) | |
| ((long)(b[off] & 0xff) )) & 0xffffffff); |
| } |
| |
| } |
