| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.io; |
| |
| import static org.mockito.Mockito.*; |
| |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.List; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.LocalFileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.io.IOUtils; |
| import org.apache.hadoop.io.SequenceFile.CompressionType; |
| import org.apache.hadoop.io.compress.CompressionCodec; |
| import org.apache.hadoop.io.compress.CompressionInputStream; |
| import org.apache.hadoop.io.compress.CompressionOutputStream; |
| import org.apache.hadoop.io.compress.Compressor; |
| import org.apache.hadoop.io.compress.Decompressor; |
| import org.apache.hadoop.test.GenericTestUtils; |
| import org.apache.hadoop.util.Progressable; |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertTrue; |
| import static org.junit.Assert.fail; |
| import static org.junit.Assert.assertNotNull; |
| import static org.junit.Assert.assertNull; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| public class TestBloomMapFile { |
| private static Configuration conf = new Configuration(); |
| private static final Path TEST_ROOT = new Path(GenericTestUtils.getTempPath( |
| TestMapFile.class.getSimpleName())); |
| private static final Path TEST_DIR = new Path(TEST_ROOT, "testfile"); |
| private static final Path TEST_FILE = new Path(TEST_ROOT, "testfile"); |
| |
| @Before |
| public void setUp() throws Exception { |
| LocalFileSystem fs = FileSystem.getLocal(conf); |
| if (fs.exists(TEST_ROOT) && !fs.delete(TEST_ROOT, true)) { |
| fail("Can't clean up test root dir"); |
| } |
| fs.mkdirs(TEST_ROOT); |
| } |
| |
| @SuppressWarnings("deprecation") |
| @Test |
| public void testMembershipTest() throws Exception { |
| // write the file |
| FileSystem fs = FileSystem.getLocal(conf); |
| Path qualifiedDirName = fs.makeQualified(TEST_DIR); |
| conf.setInt("io.mapfile.bloom.size", 2048); |
| BloomMapFile.Writer writer = null; |
| BloomMapFile.Reader reader = null; |
| try { |
| writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), |
| IntWritable.class, Text.class); |
| IntWritable key = new IntWritable(); |
| Text value = new Text(); |
| for (int i = 0; i < 2000; i += 2) { |
| key.set(i); |
| value.set("00" + i); |
| writer.append(key, value); |
| } |
| writer.close(); |
| |
| reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); |
| // check false positives rate |
| int falsePos = 0; |
| int falseNeg = 0; |
| for (int i = 0; i < 2000; i++) { |
| key.set(i); |
| boolean exists = reader.probablyHasKey(key); |
| if (i % 2 == 0) { |
| if (!exists) |
| falseNeg++; |
| } else { |
| if (exists) |
| falsePos++; |
| } |
| } |
| reader.close(); |
| fs.delete(qualifiedDirName, true); |
| System.out.println("False negatives: " + falseNeg); |
| assertEquals(0, falseNeg); |
| System.out.println("False positives: " + falsePos); |
| assertTrue(falsePos < 2); |
| } finally { |
| IOUtils.cleanup(null, writer, reader); |
| } |
| } |
| |
| @SuppressWarnings("deprecation") |
| private void checkMembershipVaryingSizedKeys(List<Text> keys) |
| throws Exception { |
| FileSystem fs = FileSystem.getLocal(conf); |
| Path qualifiedDirName = fs.makeQualified(TEST_DIR); |
| BloomMapFile.Writer writer = null; |
| BloomMapFile.Reader reader = null; |
| try { |
| writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), |
| Text.class, NullWritable.class); |
| for (Text key : keys) { |
| writer.append(key, NullWritable.get()); |
| } |
| writer.close(); |
| |
| // will check for membership in opposite order of how keys were inserted |
| reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); |
| Collections.reverse(keys); |
| for (Text key : keys) { |
| assertTrue("False negative for existing key " + key, |
| reader.probablyHasKey(key)); |
| } |
| reader.close(); |
| fs.delete(qualifiedDirName, true); |
| } finally { |
| IOUtils.cleanup(null, writer, reader); |
| } |
| } |
| |
| @Test |
| public void testMembershipVaryingSizedKeysTest1() throws Exception { |
| ArrayList<Text> list = new ArrayList<Text>(); |
| list.add(new Text("A")); |
| list.add(new Text("BB")); |
| checkMembershipVaryingSizedKeys(list); |
| } |
| |
| @Test |
| public void testMembershipVaryingSizedKeysTest2() throws Exception { |
| ArrayList<Text> list = new ArrayList<Text>(); |
| list.add(new Text("AA")); |
| list.add(new Text("B")); |
| checkMembershipVaryingSizedKeys(list); |
| } |
| |
| /** |
| * test {@code BloomMapFile.delete()} method |
| */ |
| @Test |
| public void testDeleteFile() { |
| BloomMapFile.Writer writer = null; |
| try { |
| FileSystem fs = FileSystem.getLocal(conf); |
| writer = new BloomMapFile.Writer(conf, TEST_FILE, |
| MapFile.Writer.keyClass(IntWritable.class), |
| MapFile.Writer.valueClass(Text.class)); |
| assertNotNull("testDeleteFile error !!!", writer); |
| writer.close(); |
| BloomMapFile.delete(fs, TEST_FILE.toString()); |
| } catch (Exception ex) { |
| fail("unexpect ex in testDeleteFile !!!"); |
| } finally { |
| IOUtils.cleanup(null, writer); |
| } |
| } |
| |
| /** |
| * test {@link BloomMapFile.Reader} constructor with |
| * IOException |
| */ |
| @Test |
| public void testIOExceptionInWriterConstructor() { |
| Path dirNameSpy = spy(TEST_FILE); |
| BloomMapFile.Reader reader = null; |
| BloomMapFile.Writer writer = null; |
| try { |
| writer = new BloomMapFile.Writer(conf, TEST_FILE, |
| MapFile.Writer.keyClass(IntWritable.class), |
| MapFile.Writer.valueClass(Text.class)); |
| writer.append(new IntWritable(1), new Text("123124142")); |
| writer.close(); |
| |
| when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); |
| reader = new BloomMapFile.Reader(dirNameSpy, conf, |
| MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); |
| |
| assertNull("testIOExceptionInWriterConstructor error !!!", |
| reader.getBloomFilter()); |
| } catch (Exception ex) { |
| fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); |
| } finally { |
| IOUtils.cleanup(null, writer, reader); |
| } |
| } |
| |
| /** |
| * test {@link BloomMapFile.Reader#get(WritableComparable, Writable)} method |
| */ |
| @Test |
| public void testGetBloomMapFile() { |
| int SIZE = 10; |
| BloomMapFile.Reader reader = null; |
| BloomMapFile.Writer writer = null; |
| try { |
| writer = new BloomMapFile.Writer(conf, TEST_FILE, |
| MapFile.Writer.keyClass(IntWritable.class), |
| MapFile.Writer.valueClass(Text.class)); |
| |
| for (int i = 0; i < SIZE; i++) { |
| writer.append(new IntWritable(i), new Text()); |
| } |
| writer.close(); |
| |
| reader = new BloomMapFile.Reader(TEST_FILE, conf, |
| MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); |
| |
| for (int i = 0; i < SIZE; i++) { |
| assertNotNull("testGetBloomMapFile error !!!", |
| reader.get(new IntWritable(i), new Text())); |
| } |
| |
| assertNull("testGetBloomMapFile error !!!", |
| reader.get(new IntWritable(SIZE + 5), new Text())); |
| } catch (Exception ex) { |
| fail("unexpect ex in testGetBloomMapFile !!!"); |
| } finally { |
| IOUtils.cleanup(null, writer, reader); |
| } |
| } |
| |
| /** |
| * test {@code BloomMapFile.Writer} constructors |
| */ |
| @SuppressWarnings("deprecation") |
| @Test |
| public void testBloomMapFileConstructors() { |
| BloomMapFile.Writer writer = null; |
| try { |
| FileSystem ts = FileSystem.get(conf); |
| String testFileName = TEST_FILE.toString(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, |
| defaultCodec, defaultProgress); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, |
| defaultProgress); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.RECORD, |
| defaultCodec, defaultProgress); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.RECORD, |
| defaultProgress); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, IntWritable.class, Text.class, CompressionType.RECORD); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| writer = new BloomMapFile.Writer(conf, ts, |
| testFileName, WritableComparator.get(Text.class), Text.class); |
| assertNotNull("testBloomMapFileConstructors error !!!", writer); |
| writer.close(); |
| } catch (Exception ex) { |
| fail("testBloomMapFileConstructors error !!!"); |
| } finally { |
| IOUtils.cleanup(null, writer); |
| } |
| } |
| |
| static final Progressable defaultProgress = new Progressable() { |
| @Override |
| public void progress() { |
| } |
| }; |
| |
| static final CompressionCodec defaultCodec = new CompressionCodec() { |
| @Override |
| public String getDefaultExtension() { |
| return null; |
| } |
| |
| @Override |
| public Class<? extends Decompressor> getDecompressorType() { |
| return null; |
| } |
| |
| @Override |
| public Class<? extends Compressor> getCompressorType() { |
| return null; |
| } |
| |
| @Override |
| public CompressionOutputStream createOutputStream(OutputStream out, |
| Compressor compressor) throws IOException { |
| return mock(CompressionOutputStream.class); |
| } |
| |
| @Override |
| public CompressionOutputStream createOutputStream(OutputStream out) |
| throws IOException { |
| return mock(CompressionOutputStream.class); |
| } |
| |
| @Override |
| public CompressionInputStream createInputStream(InputStream in, |
| Decompressor decompressor) throws IOException { |
| return null; |
| } |
| |
| @Override |
| public CompressionInputStream createInputStream(InputStream in) |
| throws IOException { |
| return null; |
| } |
| |
| @Override |
| public Decompressor createDecompressor() { |
| return null; |
| } |
| |
| @Override |
| public Compressor createCompressor() { |
| return null; |
| } |
| }; |
| } |