test/unit/org/apache/cassandra/io/compress/CompressorTest.java - cassandra - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.cassandra.io.compress;

 import java.io.*;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Random;

 import com.google.common.io.Files;
 import static org.junit.Assert.*;
 import org.junit.Assert;
 import org.junit.Test;

 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.utils.ByteBufferUtil;

 public class CompressorTest
 {
     ICompressor compressor;

     ICompressor[] compressors = new ICompressor[] {
             LZ4Compressor.create(Collections.<String, String>emptyMap()),
             DeflateCompressor.create(Collections.<String, String>emptyMap()),
             SnappyCompressor.create(Collections.<String, String>emptyMap())
     };

     @Test
     public void testAllCompressors() throws IOException
     {
         for (ICompressor compressor : compressors)
         {
             this.compressor = compressor;

             testEmptyArray();
             testLongArray();
             testShortArray();
             testMappedFile();
         }
     }

     public void testArrayUncompress(byte[] data, int off, int len) throws IOException
     {
         final int inOffset = 2;
         ByteBuffer src = makeBB(len + inOffset);
         src.position(inOffset);
         src.put(data, off, len);
         src.flip().position(inOffset);

         final int outOffset = 3;
         final ByteBuffer compressed = makeBB(outOffset + compressor.initialCompressedBufferLength(len));
         fillBBWithRandom(compressed);
         compressed.position(outOffset);

         compressor.compress(src, compressed);
         compressed.flip().position(outOffset);

         final int restoreOffset = 5;
         final byte[] restored = new byte[restoreOffset + len];
         new Random().nextBytes(restored);

         // need byte[] representation which direct buffers don't have
         byte[] compressedBytes = new byte[compressed.capacity()];
         ByteBufferUtil.arrayCopy(compressed, outOffset, compressedBytes, outOffset, compressed.limit() - outOffset);

         final int decompressedLength = compressor.uncompress(compressedBytes, outOffset, compressed.remaining(), restored, restoreOffset);

         assertEquals(decompressedLength, len);
         assertArrayEquals(Arrays.copyOfRange(data, off, off + len),
                           Arrays.copyOfRange(restored, restoreOffset, restoreOffset + decompressedLength));
     }

     public void testArrayUncompress(byte[] data) throws IOException
     {
         testArrayUncompress(data, 0, data.length);
     }

     public void testEmptyArray() throws IOException
     {
         testArrayUncompress(new byte[0]);
     }

     public void testShortArray() throws UnsupportedEncodingException, IOException
     {
         testArrayUncompress("Cassandra".getBytes("UTF-8"), 1, 7);
     }

     public void testLongArray() throws UnsupportedEncodingException, IOException
     {
         byte[] data = new byte[1 << 20];
         testArrayUncompress(data, 13, 1 << 19);
         new Random(0).nextBytes(data);
         testArrayUncompress(data, 13, 1 << 19);
     }

     public void testMappedFile() throws IOException
     {
         byte[] data = new byte[1 << 20];
         new Random().nextBytes(data);
         ByteBuffer src = makeBB(data.length);
         src.put(data);
         src.flip();

         // create a temp file
         File temp = File.createTempFile("tempfile", ".tmp");
         temp.deleteOnExit();

         // Prepend some random bytes to the output and compress
         final int outOffset = 3;
         byte[] garbage = new byte[outOffset + compressor.initialCompressedBufferLength(data.length)];
         new Random().nextBytes(garbage);
         ByteBuffer dest = makeBB(outOffset + compressor.initialCompressedBufferLength(data.length));
         dest.put(garbage);
         dest.clear();
         dest.position(outOffset);

         compressor.compress(src, dest);
         int compressedLength = dest.position() - outOffset;

         FileChannel channel = FileChannel.open(temp.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE);
         dest.clear();
         channel.write(dest);

         MappedByteBuffer mappedData = Files.map(temp);
         ByteBuffer result = makeBB(data.length + 100);
         mappedData.position(outOffset).limit(outOffset + compressedLength);

         compressor.uncompress(mappedData, result);
         channel.close();
         result.flip();

         Assert.assertEquals(data.length, result.limit());
         for (int i = 0; i < result.limit(); i++)
         {
             Assert.assertEquals("Decompression mismatch at byte "+i, data[i], result.get());
         }
     }

     @Test
     public void testLZ4ByteBuffers() throws IOException
     {
         compressor = LZ4Compressor.create(Collections.<String, String>emptyMap());
         testByteBuffers();
     }

     @Test
     public void testDeflateByteBuffers() throws IOException
     {
         compressor = DeflateCompressor.create(Collections.<String, String>emptyMap());
         testByteBuffers();
     }

     @Test
     public void testSnappyByteBuffers() throws IOException
     {
         compressor = SnappyCompressor.create(Collections.<String, String>emptyMap());
         testByteBuffers();
     }

     private void testByteBuffers() throws IOException
     {
         assert compressor.supports(BufferType.OFF_HEAP);
         assert compressor.supports(compressor.preferredBufferType());

         for (BufferType in: BufferType.values())
             if (compressor.supports(in))
                 for (BufferType comp: BufferType.values())
                     if (compressor.supports(comp))
                         for (BufferType out: BufferType.values())
                             if (compressor.supports(out))
                                 testByteBuffers(in, comp, out);
     }

     private void testByteBuffers(BufferType typeIn, BufferType typeComp, BufferType typeOut) throws IOException
     {
         try
         {
             int n = RandomAccessReader.DEFAULT_BUFFER_SIZE;
             byte[] srcData = new byte[n];
             new Random().nextBytes(srcData);

             final int inOffset = 2;
             ByteBuffer src = typeIn.allocate(inOffset + n + inOffset);
             src.position(inOffset);
             src.put(srcData, 0, n);
             src.flip().position(inOffset);

             int outOffset = 5;
             ByteBuffer compressed = typeComp.allocate(outOffset + compressor.initialCompressedBufferLength(srcData.length) + outOffset);
             byte[] garbage = new byte[compressed.capacity()];
             new Random().nextBytes(garbage);
             compressed.put(garbage);
             compressed.position(outOffset).limit(compressed.capacity() - outOffset);

             compressor.compress(src, compressed);
             assertEquals(inOffset + n, src.position());
             assertEquals(inOffset + n, src.limit());
             assertEquals(compressed.capacity() - outOffset, compressed.limit());
             compressed.flip().position(outOffset);
             int len = compressed.remaining();

             ByteBuffer result = typeOut.allocate(inOffset + n + inOffset);
             result.position(inOffset).limit(result.capacity() - inOffset);
             compressor.uncompress(compressed, result);
             assertEquals(outOffset + len, compressed.position());
             assertEquals(outOffset + len, compressed.limit());
             assertEquals(result.capacity() - inOffset, result.limit());

             int decompressed = result.position() - inOffset;
             assert decompressed == n : "Failed uncompressed size";
             for (int i = 0; i < n; ++i)
                 assert srcData[i] == result.get(inOffset + i) : "Failed comparison on index: " + i;
         }
         catch (Throwable e)
         {
             throw new AssertionError("Failed testing compressor " + compressor.getClass().getSimpleName() + " with buffer types in:" + typeIn + " compressed:" + typeComp + " out:" + typeOut, e);
         }
     }

     private ByteBuffer makeBB(int size)
     {
         return compressor.preferredBufferType().allocate(size);
     }

     private void fillBBWithRandom(ByteBuffer dest)
     {
         byte[] random = new byte[dest.capacity()];
         new Random().nextBytes(random);
         dest.clear();
         dest.put(random);
     }

 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.cassandra.io.compress;

	import java.io.*;
	import java.nio.ByteBuffer;
	import java.nio.MappedByteBuffer;
	import java.nio.channels.FileChannel;
	import java.nio.file.StandardOpenOption;
	import java.util.Arrays;
	import java.util.Collections;
	import java.util.Random;

	import com.google.common.io.Files;
	import static org.junit.Assert.*;
	import org.junit.Assert;
	import org.junit.Test;

	import org.apache.cassandra.io.util.RandomAccessReader;
	import org.apache.cassandra.utils.ByteBufferUtil;

	public class CompressorTest
	{
	ICompressor compressor;

	ICompressor[] compressors = new ICompressor[] {
	LZ4Compressor.create(Collections.<String, String>emptyMap()),
	DeflateCompressor.create(Collections.<String, String>emptyMap()),
	SnappyCompressor.create(Collections.<String, String>emptyMap())
	};

	@Test
	public void testAllCompressors() throws IOException
	{
	for (ICompressor compressor : compressors)
	{
	this.compressor = compressor;

	testEmptyArray();
	testLongArray();
	testShortArray();
	testMappedFile();
	}
	}

	public void testArrayUncompress(byte[] data, int off, int len) throws IOException
	{
	final int inOffset = 2;
	ByteBuffer src = makeBB(len + inOffset);
	src.position(inOffset);
	src.put(data, off, len);
	src.flip().position(inOffset);

	final int outOffset = 3;
	final ByteBuffer compressed = makeBB(outOffset + compressor.initialCompressedBufferLength(len));
	fillBBWithRandom(compressed);
	compressed.position(outOffset);

	compressor.compress(src, compressed);
	compressed.flip().position(outOffset);

	final int restoreOffset = 5;
	final byte[] restored = new byte[restoreOffset + len];
	new Random().nextBytes(restored);

	// need byte[] representation which direct buffers don't have
	byte[] compressedBytes = new byte[compressed.capacity()];
	ByteBufferUtil.arrayCopy(compressed, outOffset, compressedBytes, outOffset, compressed.limit() - outOffset);

	final int decompressedLength = compressor.uncompress(compressedBytes, outOffset, compressed.remaining(), restored, restoreOffset);

	assertEquals(decompressedLength, len);
	assertArrayEquals(Arrays.copyOfRange(data, off, off + len),
	Arrays.copyOfRange(restored, restoreOffset, restoreOffset + decompressedLength));
	}

	public void testArrayUncompress(byte[] data) throws IOException
	{
	testArrayUncompress(data, 0, data.length);
	}

	public void testEmptyArray() throws IOException
	{
	testArrayUncompress(new byte[0]);
	}

	public void testShortArray() throws UnsupportedEncodingException, IOException
	{
	testArrayUncompress("Cassandra".getBytes("UTF-8"), 1, 7);
	}

	public void testLongArray() throws UnsupportedEncodingException, IOException
	{
	byte[] data = new byte[1 << 20];
	testArrayUncompress(data, 13, 1 << 19);
	new Random(0).nextBytes(data);
	testArrayUncompress(data, 13, 1 << 19);
	}

	public void testMappedFile() throws IOException
	{
	byte[] data = new byte[1 << 20];
	new Random().nextBytes(data);
	ByteBuffer src = makeBB(data.length);
	src.put(data);
	src.flip();

	// create a temp file
	File temp = File.createTempFile("tempfile", ".tmp");
	temp.deleteOnExit();

	// Prepend some random bytes to the output and compress
	final int outOffset = 3;
	byte[] garbage = new byte[outOffset + compressor.initialCompressedBufferLength(data.length)];
	new Random().nextBytes(garbage);
	ByteBuffer dest = makeBB(outOffset + compressor.initialCompressedBufferLength(data.length));
	dest.put(garbage);
	dest.clear();
	dest.position(outOffset);

	compressor.compress(src, dest);
	int compressedLength = dest.position() - outOffset;

	FileChannel channel = FileChannel.open(temp.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE);
	dest.clear();
	channel.write(dest);

	MappedByteBuffer mappedData = Files.map(temp);
	ByteBuffer result = makeBB(data.length + 100);
	mappedData.position(outOffset).limit(outOffset + compressedLength);

	compressor.uncompress(mappedData, result);
	channel.close();
	result.flip();

	Assert.assertEquals(data.length, result.limit());
	for (int i = 0; i < result.limit(); i++)
	{
	Assert.assertEquals("Decompression mismatch at byte "+i, data[i], result.get());
	}
	}

	@Test
	public void testLZ4ByteBuffers() throws IOException
	{
	compressor = LZ4Compressor.create(Collections.<String, String>emptyMap());
	testByteBuffers();
	}

	@Test
	public void testDeflateByteBuffers() throws IOException
	{
	compressor = DeflateCompressor.create(Collections.<String, String>emptyMap());
	testByteBuffers();
	}

	@Test
	public void testSnappyByteBuffers() throws IOException
	{
	compressor = SnappyCompressor.create(Collections.<String, String>emptyMap());
	testByteBuffers();
	}

	private void testByteBuffers() throws IOException
	{
	assert compressor.supports(BufferType.OFF_HEAP);
	assert compressor.supports(compressor.preferredBufferType());

	for (BufferType in: BufferType.values())
	if (compressor.supports(in))
	for (BufferType comp: BufferType.values())
	if (compressor.supports(comp))
	for (BufferType out: BufferType.values())
	if (compressor.supports(out))
	testByteBuffers(in, comp, out);
	}

	private void testByteBuffers(BufferType typeIn, BufferType typeComp, BufferType typeOut) throws IOException
	{
	try
	{
	int n = RandomAccessReader.DEFAULT_BUFFER_SIZE;
	byte[] srcData = new byte[n];
	new Random().nextBytes(srcData);

	final int inOffset = 2;
	ByteBuffer src = typeIn.allocate(inOffset + n + inOffset);
	src.position(inOffset);
	src.put(srcData, 0, n);
	src.flip().position(inOffset);

	int outOffset = 5;
	ByteBuffer compressed = typeComp.allocate(outOffset + compressor.initialCompressedBufferLength(srcData.length) + outOffset);
	byte[] garbage = new byte[compressed.capacity()];
	new Random().nextBytes(garbage);
	compressed.put(garbage);
	compressed.position(outOffset).limit(compressed.capacity() - outOffset);

	compressor.compress(src, compressed);
	assertEquals(inOffset + n, src.position());
	assertEquals(inOffset + n, src.limit());
	assertEquals(compressed.capacity() - outOffset, compressed.limit());
	compressed.flip().position(outOffset);
	int len = compressed.remaining();

	ByteBuffer result = typeOut.allocate(inOffset + n + inOffset);
	result.position(inOffset).limit(result.capacity() - inOffset);
	compressor.uncompress(compressed, result);
	assertEquals(outOffset + len, compressed.position());
	assertEquals(outOffset + len, compressed.limit());
	assertEquals(result.capacity() - inOffset, result.limit());

	int decompressed = result.position() - inOffset;
	assert decompressed == n : "Failed uncompressed size";
	for (int i = 0; i < n; ++i)
	assert srcData[i] == result.get(inOffset + i) : "Failed comparison on index: " + i;
	}
	catch (Throwable e)
	{
	throw new AssertionError("Failed testing compressor " + compressor.getClass().getSimpleName() + " with buffer types in:" + typeIn + " compressed:" + typeComp + " out:" + typeOut, e);
	}
	}

	private ByteBuffer makeBB(int size)
	{
	return compressor.preferredBufferType().allocate(size);
	}

	private void fillBBWithRandom(ByteBuffer dest)
	{
	byte[] random = new byte[dest.capacity()];
	new Random().nextBytes(random);
	dest.clear();
	dest.put(random);
	}

	}