| package org.apache.lucene.store; |
| |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import java.io.EOFException; |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.nio.file.Files; |
| import java.nio.file.NoSuchFileException; |
| import java.nio.file.Path; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.zip.CRC32; |
| |
| import org.apache.lucene.index.DirectoryReader; |
| import org.apache.lucene.index.IndexNotFoundException; |
| import org.apache.lucene.util.IOUtils; |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| /** Base class for per-Directory tests. */ |
| |
| public abstract class BaseDirectoryTestCase extends LuceneTestCase { |
| |
| /** Subclass returns the Directory to be tested; if it's |
| * an FS-based directory it should point to the specified |
| * path, else it can ignore it. */ |
| protected abstract Directory getDirectory(Path path) throws IOException; |
| |
| // first some basic tests for the directory api |
| |
| public void testCopyFrom() throws Exception { |
| Directory source = getDirectory(createTempDir("testCopy")); |
| Directory dest = newDirectory(); |
| |
| IndexOutput output = source.createOutput("foobar", newIOContext(random())); |
| int numBytes = random().nextInt(20000); |
| byte bytes[] = new byte[numBytes]; |
| random().nextBytes(bytes); |
| output.writeBytes(bytes, bytes.length); |
| output.close(); |
| |
| dest.copyFrom(source, "foobar", "foobaz", newIOContext(random())); |
| assertTrue(slowFileExists(dest, "foobaz")); |
| |
| IndexInput input = dest.openInput("foobaz", newIOContext(random())); |
| byte bytes2[] = new byte[numBytes]; |
| input.readBytes(bytes2, 0, bytes2.length); |
| assertEquals(input.length(), numBytes); |
| input.close(); |
| |
| assertArrayEquals(bytes, bytes2); |
| |
| IOUtils.close(source, dest); |
| } |
| |
| public void testCopyFromDestination() throws Exception { |
| Directory source = newDirectory(); |
| Directory dest = getDirectory(createTempDir("testCopyDestination")); |
| |
| IndexOutput output = source.createOutput("foobar", newIOContext(random())); |
| int numBytes = random().nextInt(20000); |
| byte bytes[] = new byte[numBytes]; |
| random().nextBytes(bytes); |
| output.writeBytes(bytes, bytes.length); |
| output.close(); |
| |
| dest.copyFrom(source, "foobar", "foobaz", newIOContext(random())); |
| assertTrue(slowFileExists(dest, "foobaz")); |
| |
| IndexInput input = dest.openInput("foobaz", newIOContext(random())); |
| byte bytes2[] = new byte[numBytes]; |
| input.readBytes(bytes2, 0, bytes2.length); |
| assertEquals(input.length(), numBytes); |
| input.close(); |
| |
| assertArrayEquals(bytes, bytes2); |
| |
| IOUtils.close(source, dest); |
| } |
| |
| public void testRename() throws Exception { |
| Directory dir = getDirectory(createTempDir("testRename")); |
| |
| IndexOutput output = dir.createOutput("foobar", newIOContext(random())); |
| int numBytes = random().nextInt(20000); |
| byte bytes[] = new byte[numBytes]; |
| random().nextBytes(bytes); |
| output.writeBytes(bytes, bytes.length); |
| output.close(); |
| |
| dir.renameFile("foobar", "foobaz"); |
| |
| IndexInput input = dir.openInput("foobaz", newIOContext(random())); |
| byte bytes2[] = new byte[numBytes]; |
| input.readBytes(bytes2, 0, bytes2.length); |
| assertEquals(input.length(), numBytes); |
| input.close(); |
| |
| assertArrayEquals(bytes, bytes2); |
| |
| dir.close(); |
| } |
| |
| // TODO: are these semantics really needed by lucene? can we just throw exception? |
| public void testCopyOverwrite() throws Exception { |
| Directory source = getDirectory(createTempDir("testCopyOverwrite")); |
| Directory dest = newDirectory(); |
| |
| // we are double-writing intentionally, because thats the api |
| if (dest instanceof MockDirectoryWrapper) { |
| ((MockDirectoryWrapper) dest).setPreventDoubleWrite(false); |
| } |
| |
| IndexOutput output = source.createOutput("foobar", newIOContext(random())); |
| int numBytes = random().nextInt(20000); |
| byte bytes[] = new byte[numBytes]; |
| random().nextBytes(bytes); |
| output.writeBytes(bytes, bytes.length); |
| output.close(); |
| |
| // create foobaz first, it should be overwritten |
| IndexOutput output2 = dest.createOutput("foobaz", newIOContext(random())); |
| output2.writeString("bogus!"); |
| output2.close(); |
| |
| dest.copyFrom(source, "foobar", "foobaz", newIOContext(random())); |
| assertTrue(slowFileExists(dest, "foobaz")); |
| |
| IndexInput input = dest.openInput("foobaz", newIOContext(random())); |
| byte bytes2[] = new byte[numBytes]; |
| input.readBytes(bytes2, 0, bytes2.length); |
| assertEquals(input.length(), numBytes); |
| input.close(); |
| |
| assertArrayEquals(bytes, bytes2); |
| |
| IOUtils.close(source, dest); |
| } |
| |
| public void testDeleteFile() throws Exception { |
| Directory dir = getDirectory(createTempDir("testDeleteFile")); |
| int count = dir.listAll().length; |
| dir.createOutput("foo.txt", IOContext.DEFAULT).close(); |
| assertEquals(count+1, dir.listAll().length); |
| dir.deleteFiles(Collections.singleton("foo.txt")); |
| assertEquals(count, dir.listAll().length); |
| dir.close(); |
| } |
| |
| public void testByte() throws Exception { |
| Directory dir = getDirectory(createTempDir("testByte")); |
| IndexOutput output = dir.createOutput("byte", newIOContext(random())); |
| output.writeByte((byte)128); |
| output.close(); |
| |
| IndexInput input = dir.openInput("byte", newIOContext(random())); |
| assertEquals(1, input.length()); |
| assertEquals((byte)128, input.readByte()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testShort() throws Exception { |
| Directory dir = getDirectory(createTempDir("testShort")); |
| IndexOutput output = dir.createOutput("short", newIOContext(random())); |
| output.writeShort((short)-20); |
| output.close(); |
| |
| IndexInput input = dir.openInput("short", newIOContext(random())); |
| assertEquals(2, input.length()); |
| assertEquals((short)-20, input.readShort()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testInt() throws Exception { |
| Directory dir = getDirectory(createTempDir("testInt")); |
| IndexOutput output = dir.createOutput("int", newIOContext(random())); |
| output.writeInt(-500); |
| output.close(); |
| |
| IndexInput input = dir.openInput("int", newIOContext(random())); |
| assertEquals(4, input.length()); |
| assertEquals(-500, input.readInt()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testLong() throws Exception { |
| Directory dir = getDirectory(createTempDir("testLong")); |
| IndexOutput output = dir.createOutput("long", newIOContext(random())); |
| output.writeLong(-5000); |
| output.close(); |
| |
| IndexInput input = dir.openInput("long", newIOContext(random())); |
| assertEquals(8, input.length()); |
| assertEquals(-5000L, input.readLong()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testString() throws Exception { |
| Directory dir = getDirectory(createTempDir("testString")); |
| IndexOutput output = dir.createOutput("string", newIOContext(random())); |
| output.writeString("hello!"); |
| output.close(); |
| |
| IndexInput input = dir.openInput("string", newIOContext(random())); |
| assertEquals("hello!", input.readString()); |
| assertEquals(7, input.length()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testVInt() throws Exception { |
| Directory dir = getDirectory(createTempDir("testVInt")); |
| IndexOutput output = dir.createOutput("vint", newIOContext(random())); |
| output.writeVInt(500); |
| output.close(); |
| |
| IndexInput input = dir.openInput("vint", newIOContext(random())); |
| assertEquals(2, input.length()); |
| assertEquals(500, input.readVInt()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testVLong() throws Exception { |
| Directory dir = getDirectory(createTempDir("testVLong")); |
| IndexOutput output = dir.createOutput("vlong", newIOContext(random())); |
| output.writeVLong(Long.MAX_VALUE); |
| output.close(); |
| |
| IndexInput input = dir.openInput("vlong", newIOContext(random())); |
| assertEquals(9, input.length()); |
| assertEquals(Long.MAX_VALUE, input.readVLong()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testZInt() throws Exception { |
| final int[] ints = new int[random().nextInt(10)]; |
| for (int i = 0; i < ints.length; ++i) { |
| switch (random().nextInt(3)) { |
| case 0: |
| ints[i] = random().nextInt(); |
| break; |
| case 1: |
| ints[i] = random().nextBoolean() ? Integer.MIN_VALUE : Integer.MAX_VALUE; |
| break; |
| case 2: |
| ints[i] = (random().nextBoolean() ? -1 : 1) * random().nextInt(1024); |
| break; |
| default: |
| throw new AssertionError(); |
| } |
| } |
| Directory dir = getDirectory(createTempDir("testZInt")); |
| IndexOutput output = dir.createOutput("zint", newIOContext(random())); |
| for (int i : ints) { |
| output.writeZInt(i); |
| } |
| output.close(); |
| |
| IndexInput input = dir.openInput("zint", newIOContext(random())); |
| for (int i : ints) { |
| assertEquals(i, input.readZInt()); |
| } |
| assertEquals(input.length(), input.getFilePointer()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testZLong() throws Exception { |
| final long[] longs = new long[random().nextInt(10)]; |
| for (int i = 0; i < longs.length; ++i) { |
| switch (random().nextInt(3)) { |
| case 0: |
| longs[i] = random().nextLong(); |
| break; |
| case 1: |
| longs[i] = random().nextBoolean() ? Long.MIN_VALUE : Long.MAX_VALUE; |
| break; |
| case 2: |
| longs[i] = (random().nextBoolean() ? -1 : 1) * random().nextInt(1024); |
| break; |
| default: |
| throw new AssertionError(); |
| } |
| } |
| Directory dir = getDirectory(createTempDir("testZLong")); |
| IndexOutput output = dir.createOutput("zlong", newIOContext(random())); |
| for (long l : longs) { |
| output.writeZLong(l); |
| } |
| output.close(); |
| |
| IndexInput input = dir.openInput("zlong", newIOContext(random())); |
| for (long l : longs) { |
| assertEquals(l, input.readZLong()); |
| } |
| assertEquals(input.length(), input.getFilePointer()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testStringSet() throws Exception { |
| Directory dir = getDirectory(createTempDir("testStringSet")); |
| IndexOutput output = dir.createOutput("stringset", newIOContext(random())); |
| output.writeStringSet(asSet("test1", "test2")); |
| output.close(); |
| |
| IndexInput input = dir.openInput("stringset", newIOContext(random())); |
| assertEquals(16, input.length()); |
| assertEquals(asSet("test1", "test2"), input.readStringSet()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testStringMap() throws Exception { |
| Map<String,String> m = new HashMap<>(); |
| m.put("test1", "value1"); |
| m.put("test2", "value2"); |
| |
| Directory dir = getDirectory(createTempDir("testStringMap")); |
| IndexOutput output = dir.createOutput("stringmap", newIOContext(random())); |
| output.writeStringStringMap(m); |
| output.close(); |
| |
| IndexInput input = dir.openInput("stringmap", newIOContext(random())); |
| assertEquals(30, input.length()); |
| assertEquals(m, input.readStringStringMap()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testSetOfStrings() throws Exception { |
| Directory dir = getDirectory(createTempDir("testSetOfStrings")); |
| |
| IndexOutput output = dir.createOutput("stringset", newIOContext(random())); |
| output.writeSetOfStrings(asSet("test1", "test2")); |
| output.writeSetOfStrings(Collections.emptySet()); |
| output.writeSetOfStrings(asSet("test3")); |
| output.close(); |
| |
| IndexInput input = dir.openInput("stringset", newIOContext(random())); |
| Set<String> set = input.readSetOfStrings(); |
| assertEquals(asSet("test1", "test2"), set); |
| try { |
| set.add("bogus"); |
| fail("set should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| set = input.readSetOfStrings(); |
| assertEquals(Collections.emptySet(), set); |
| try { |
| set.add("bogus"); |
| fail("set should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| set = input.readSetOfStrings(); |
| assertEquals(Collections.singleton("test3"), set); |
| try { |
| set.add("bogus"); |
| fail("set should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| assertEquals(input.length(), input.getFilePointer()); |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testMapOfStrings() throws Exception { |
| Map<String,String> m = new HashMap<>(); |
| m.put("test1", "value1"); |
| m.put("test2", "value2"); |
| |
| Directory dir = getDirectory(createTempDir("testMapOfStrings")); |
| IndexOutput output = dir.createOutput("stringmap", newIOContext(random())); |
| output.writeMapOfStrings(m); |
| output.writeMapOfStrings(Collections.emptyMap()); |
| output.writeMapOfStrings(Collections.singletonMap("key", "value")); |
| output.close(); |
| |
| IndexInput input = dir.openInput("stringmap", newIOContext(random())); |
| Map<String,String> map = input.readMapOfStrings(); |
| assertEquals(m, map); |
| try { |
| map.put("bogus1", "bogus2"); |
| fail("map should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| map = input.readMapOfStrings(); |
| assertEquals(Collections.emptyMap(), map); |
| try { |
| map.put("bogus1", "bogus2"); |
| fail("map should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| map = input.readMapOfStrings(); |
| assertEquals(Collections.singletonMap("key", "value"), map); |
| try { |
| map.put("bogus1", "bogus2"); |
| fail("map should be immutable"); |
| } catch (UnsupportedOperationException expected) { |
| // ok |
| } |
| |
| assertEquals(input.length(), input.getFilePointer()); |
| input.close(); |
| dir.close(); |
| } |
| |
| // TODO: fold in some of the testing of o.a.l.index.TestIndexInput in here! |
| public void testChecksum() throws Exception { |
| CRC32 expected = new CRC32(); |
| int numBytes = random().nextInt(20000); |
| byte bytes[] = new byte[numBytes]; |
| random().nextBytes(bytes); |
| expected.update(bytes); |
| |
| Directory dir = getDirectory(createTempDir("testChecksum")); |
| IndexOutput output = dir.createOutput("checksum", newIOContext(random())); |
| output.writeBytes(bytes, 0, bytes.length); |
| output.close(); |
| |
| ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random())); |
| input.skipBytes(numBytes); |
| |
| assertEquals(expected.getValue(), input.getChecksum()); |
| input.close(); |
| dir.close(); |
| } |
| |
| /** Make sure directory throws AlreadyClosedException if |
| * you try to createOutput after closing. */ |
| public void testDetectClose() throws Throwable { |
| Directory dir = getDirectory(createTempDir("testDetectClose")); |
| dir.close(); |
| try { |
| dir.createOutput("test", newIOContext(random())); |
| fail("did not hit expected exception"); |
| } catch (AlreadyClosedException ace) { |
| // expected |
| } |
| } |
| |
| public void testThreadSafety() throws Exception { |
| final Directory dir = getDirectory(createTempDir("testThreadSafety")); |
| if (dir instanceof BaseDirectoryWrapper) { |
| ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we arent making an index |
| } |
| if (dir instanceof MockDirectoryWrapper) { |
| ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); // makes this test really slow |
| } |
| |
| if (VERBOSE) { |
| System.out.println(dir); |
| } |
| |
| class TheThread extends Thread { |
| private String name; |
| |
| public TheThread(String name) { |
| this.name = name; |
| } |
| |
| @Override |
| public void run() { |
| for (int i = 0; i < 1000; i++) { |
| String fileName = this.name + i; |
| try { |
| //System.out.println("create:" + fileName); |
| IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| output.close(); |
| assertTrue(slowFileExists(dir, fileName)); |
| } catch (IOException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| } |
| }; |
| |
| class TheThread2 extends Thread { |
| private String name; |
| private volatile boolean stop; |
| |
| public TheThread2(String name) { |
| this.name = name; |
| } |
| |
| @Override |
| public void run() { |
| while (stop == false) { |
| try { |
| String[] files = dir.listAll(); |
| for (String file : files) { |
| if (!file.startsWith(name)) { |
| continue; |
| } |
| //System.out.println("file:" + file); |
| try { |
| IndexInput input = dir.openInput(file, newIOContext(random())); |
| input.close(); |
| } catch (FileNotFoundException | NoSuchFileException e) { |
| // ignore |
| } catch (IOException e) { |
| if (e.getMessage() != null && e.getMessage().contains("still open for writing")) { |
| // ignore |
| } else { |
| throw new RuntimeException(e); |
| } |
| } |
| if (random().nextBoolean()) { |
| break; |
| } |
| } |
| } catch (IOException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| } |
| }; |
| |
| TheThread theThread = new TheThread("t1"); |
| TheThread2 theThread2 = new TheThread2("t2"); |
| theThread.start(); |
| theThread2.start(); |
| |
| theThread.join(); |
| |
| // after first thread is done, no sense in waiting on thread 2 |
| // to listFiles() and loop over and over |
| theThread2.stop = true; |
| theThread2.join(); |
| |
| dir.close(); |
| } |
| |
| /** LUCENE-1468: once we create an output, we should see |
| * it in the dir listing and be able to open it with |
| * openInput. */ |
| public void testDirectoryFilter() throws IOException { |
| String name = "file"; |
| Directory dir = getDirectory(createTempDir("testDirectoryFilter")); |
| try { |
| dir.createOutput(name, newIOContext(random())).close(); |
| assertTrue(slowFileExists(dir, name)); |
| assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| } finally { |
| dir.close(); |
| } |
| } |
| |
| // LUCENE-2852 |
| public void testSeekToEOFThenBack() throws Exception { |
| Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack")); |
| |
| IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE]; |
| o.writeBytes(bytes, 0, bytes.length); |
| o.close(); |
| |
| IndexInput i = dir.openInput("out", newIOContext(random())); |
| i.seek(2*RAMInputStream.BUFFER_SIZE-1); |
| i.seek(3*RAMInputStream.BUFFER_SIZE); |
| i.seek(RAMInputStream.BUFFER_SIZE); |
| i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE); |
| i.close(); |
| dir.close(); |
| } |
| |
| // LUCENE-1196 |
| public void testIllegalEOF() throws Exception { |
| Directory dir = getDirectory(createTempDir("testIllegalEOF")); |
| IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| byte[] b = new byte[1024]; |
| o.writeBytes(b, 0, 1024); |
| o.close(); |
| IndexInput i = dir.openInput("out", newIOContext(random())); |
| i.seek(1024); |
| i.close(); |
| dir.close(); |
| } |
| |
| public void testSeekPastEOF() throws Exception { |
| Directory dir = getDirectory(createTempDir("testSeekPastEOF")); |
| IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| final int len = random().nextInt(2048); |
| byte[] b = new byte[len]; |
| o.writeBytes(b, 0, len); |
| o.close(); |
| IndexInput i = dir.openInput("out", newIOContext(random())); |
| try { |
| i.seek(len + random().nextInt(2048)); |
| i.readByte(); |
| fail("Did not get EOFException"); |
| } catch (EOFException eof) { |
| // pass |
| } |
| i.close(); |
| dir.close(); |
| } |
| |
| public void testSliceOutOfBounds() throws Exception { |
| Directory dir = getDirectory(createTempDir("testSliceOutOfBounds")); |
| IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| final int len = random().nextInt(2040) + 8; |
| byte[] b = new byte[len]; |
| o.writeBytes(b, 0, len); |
| o.close(); |
| IndexInput i = dir.openInput("out", newIOContext(random())); |
| try { |
| i.slice("slice1", 0, len + 1); |
| fail("Did not get IllegalArgumentException"); |
| } catch (IllegalArgumentException iae) { |
| // pass |
| } |
| try { |
| i.slice("slice2", -1, len); |
| fail("Did not get IllegalArgumentException"); |
| } catch (IllegalArgumentException iae) { |
| // pass |
| } |
| IndexInput slice = i.slice("slice3", 4, len / 2); |
| try { |
| slice.slice("slice3sub", 1, len / 2); |
| fail("Did not get IllegalArgumentException"); |
| } catch (IllegalArgumentException iae) { |
| // pass |
| } |
| i.close(); |
| dir.close(); |
| } |
| |
| // LUCENE-3382 -- make sure we get exception if the directory really does not exist. |
| public void testNoDir() throws Throwable { |
| Path tempDir = createTempDir("doesnotexist"); |
| assumeFalse("test directly deletes files", TestUtil.hasVirusChecker(tempDir)); |
| IOUtils.rm(tempDir); |
| Directory dir = getDirectory(tempDir); |
| try { |
| DirectoryReader.open(dir); |
| fail("did not hit expected exception"); |
| } catch (NoSuchFileException | IndexNotFoundException nsde) { |
| // expected |
| } |
| dir.close(); |
| } |
| |
| public void testCopyBytes() throws Exception { |
| testCopyBytes(getDirectory(createTempDir("testCopyBytes"))); |
| } |
| |
| private static byte value(int idx) { |
| return (byte) ((idx % 256) * (1 + (idx / 256))); |
| } |
| |
| public static void testCopyBytes(Directory dir) throws Exception { |
| |
| // make random file |
| IndexOutput out = dir.createOutput("test", newIOContext(random())); |
| byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)]; |
| final int size = TestUtil.nextInt(random(), 1, 1777777); |
| int upto = 0; |
| int byteUpto = 0; |
| while (upto < size) { |
| bytes[byteUpto++] = value(upto); |
| upto++; |
| if (byteUpto == bytes.length) { |
| out.writeBytes(bytes, 0, bytes.length); |
| byteUpto = 0; |
| } |
| } |
| |
| out.writeBytes(bytes, 0, byteUpto); |
| assertEquals(size, out.getFilePointer()); |
| out.close(); |
| assertEquals(size, dir.fileLength("test")); |
| |
| // copy from test -> test2 |
| final IndexInput in = dir.openInput("test", newIOContext(random())); |
| |
| out = dir.createOutput("test2", newIOContext(random())); |
| |
| upto = 0; |
| while (upto < size) { |
| if (random().nextBoolean()) { |
| out.writeByte(in.readByte()); |
| upto++; |
| } else { |
| final int chunk = Math.min( |
| TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| out.copyBytes(in, chunk); |
| upto += chunk; |
| } |
| } |
| assertEquals(size, upto); |
| out.close(); |
| in.close(); |
| |
| // verify |
| IndexInput in2 = dir.openInput("test2", newIOContext(random())); |
| upto = 0; |
| while (upto < size) { |
| if (random().nextBoolean()) { |
| final byte v = in2.readByte(); |
| assertEquals(value(upto), v); |
| upto++; |
| } else { |
| final int limit = Math.min( |
| TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| in2.readBytes(bytes, 0, limit); |
| for (int byteIdx = 0; byteIdx < limit; byteIdx++) { |
| assertEquals(value(upto), bytes[byteIdx]); |
| upto++; |
| } |
| } |
| } |
| in2.close(); |
| |
| dir.deleteFiles(Arrays.asList(new String[] {"test", "test2"})); |
| dir.close(); |
| } |
| |
| // LUCENE-3541 |
| public void testCopyBytesWithThreads() throws Exception { |
| testCopyBytesWithThreads(getDirectory(createTempDir("testCopyBytesWithThreads"))); |
| } |
| |
| public static void testCopyBytesWithThreads(Directory d) throws Exception { |
| int datalen = TestUtil.nextInt(random(), 101, 10000); |
| byte data[] = new byte[datalen]; |
| random().nextBytes(data); |
| |
| IndexOutput output = d.createOutput("data", IOContext.DEFAULT); |
| output.writeBytes(data, 0, datalen); |
| output.close(); |
| |
| IndexInput input = d.openInput("data", IOContext.DEFAULT); |
| IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT); |
| // copy our 100-byte header |
| outputHeader.copyBytes(input, 100); |
| outputHeader.close(); |
| |
| // now make N copies of the remaining bytes |
| CopyThread copies[] = new CopyThread[10]; |
| for (int i = 0; i < copies.length; i++) { |
| copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT)); |
| } |
| |
| for (int i = 0; i < copies.length; i++) { |
| copies[i].start(); |
| } |
| |
| for (int i = 0; i < copies.length; i++) { |
| copies[i].join(); |
| } |
| |
| for (int i = 0; i < copies.length; i++) { |
| IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT); |
| byte[] dataCopy = new byte[datalen]; |
| System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing |
| copiedData.readBytes(dataCopy, 100, datalen-100); |
| assertArrayEquals(data, dataCopy); |
| copiedData.close(); |
| } |
| input.close(); |
| d.close(); |
| |
| } |
| |
| static class CopyThread extends Thread { |
| final IndexInput src; |
| final IndexOutput dst; |
| |
| CopyThread(IndexInput src, IndexOutput dst) { |
| this.src = src; |
| this.dst = dst; |
| } |
| |
| @Override |
| public void run() { |
| try { |
| dst.copyBytes(src, src.length()-100); |
| dst.close(); |
| } catch (IOException ex) { |
| throw new RuntimeException(ex); |
| } |
| } |
| } |
| |
| // this test backdoors the directory via the filesystem. so it must actually use the filesystem |
| // TODO: somehow change this test to |
| public void testFsyncDoesntCreateNewFiles() throws Exception { |
| Path path = createTempDir("nocreate"); |
| assumeFalse("we directly delete files", TestUtil.hasVirusChecker(path)); |
| Directory fsdir = getDirectory(path); |
| |
| // this test backdoors the directory via the filesystem. so it must be an FSDir (for now) |
| // TODO: figure a way to test this better/clean it up. E.g. we should be testing for FileSwitchDir, |
| // if it's using two FSdirs and so on |
| if (fsdir instanceof FSDirectory == false) { |
| fsdir.close(); |
| assumeTrue("test only works for FSDirectory subclasses", false); |
| } |
| |
| // write a file |
| IndexOutput out = fsdir.createOutput("afile", newIOContext(random())); |
| out.writeString("boo"); |
| out.close(); |
| |
| // delete it |
| Files.delete(path.resolve("afile")); |
| |
| int fileCount = fsdir.listAll().length; |
| |
| // fsync it |
| try { |
| fsdir.sync(Collections.singleton("afile")); |
| fail("didn't get expected exception, instead fsync created new files: " + Arrays.asList(fsdir.listAll())); |
| } catch (FileNotFoundException | NoSuchFileException expected) { |
| // ok |
| } |
| |
| // no new files created |
| assertEquals(fileCount, fsdir.listAll().length); |
| |
| fsdir.close(); |
| } |
| |
| // random access APIs |
| |
| public void testRandomLong() throws Exception { |
| Directory dir = getDirectory(createTempDir("testLongs")); |
| IndexOutput output = dir.createOutput("longs", newIOContext(random())); |
| int num = TestUtil.nextInt(random(), 50, 3000); |
| long longs[] = new long[num]; |
| for (int i = 0; i < longs.length; i++) { |
| longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE); |
| output.writeLong(longs[i]); |
| } |
| output.close(); |
| |
| // slice |
| IndexInput input = dir.openInput("longs", newIOContext(random())); |
| RandomAccessInput slice = input.randomAccessSlice(0, input.length()); |
| for (int i = 0; i < longs.length; i++) { |
| assertEquals(longs[i], slice.readLong(i * 8)); |
| } |
| |
| // subslices |
| for (int i = 1; i < longs.length; i++) { |
| long offset = i * 8; |
| RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); |
| for (int j = i; j < longs.length; j++) { |
| assertEquals(longs[j], subslice.readLong((j - i) * 8)); |
| } |
| } |
| |
| // with padding |
| for (int i = 0; i < 7; i++) { |
| String name = "longs-" + i; |
| IndexOutput o = dir.createOutput(name, newIOContext(random())); |
| byte junk[] = new byte[i]; |
| random().nextBytes(junk); |
| o.writeBytes(junk, junk.length); |
| input.seek(0); |
| o.copyBytes(input, input.length()); |
| o.close(); |
| IndexInput padded = dir.openInput(name, newIOContext(random())); |
| RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); |
| for (int j = 0; j < longs.length; j++) { |
| assertEquals(longs[j], whole.readLong(j * 8)); |
| } |
| padded.close(); |
| } |
| |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testRandomInt() throws Exception { |
| Directory dir = getDirectory(createTempDir("testInts")); |
| IndexOutput output = dir.createOutput("ints", newIOContext(random())); |
| int num = TestUtil.nextInt(random(), 50, 3000); |
| int ints[] = new int[num]; |
| for (int i = 0; i < ints.length; i++) { |
| ints[i] = random().nextInt(); |
| output.writeInt(ints[i]); |
| } |
| output.close(); |
| |
| // slice |
| IndexInput input = dir.openInput("ints", newIOContext(random())); |
| RandomAccessInput slice = input.randomAccessSlice(0, input.length()); |
| for (int i = 0; i < ints.length; i++) { |
| assertEquals(ints[i], slice.readInt(i * 4)); |
| } |
| |
| // subslices |
| for (int i = 1; i < ints.length; i++) { |
| long offset = i * 4; |
| RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); |
| for (int j = i; j < ints.length; j++) { |
| assertEquals(ints[j], subslice.readInt((j - i) * 4)); |
| } |
| } |
| |
| // with padding |
| for (int i = 0; i < 7; i++) { |
| String name = "ints-" + i; |
| IndexOutput o = dir.createOutput(name, newIOContext(random())); |
| byte junk[] = new byte[i]; |
| random().nextBytes(junk); |
| o.writeBytes(junk, junk.length); |
| input.seek(0); |
| o.copyBytes(input, input.length()); |
| o.close(); |
| IndexInput padded = dir.openInput(name, newIOContext(random())); |
| RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); |
| for (int j = 0; j < ints.length; j++) { |
| assertEquals(ints[j], whole.readInt(j * 4)); |
| } |
| padded.close(); |
| } |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testRandomShort() throws Exception { |
| Directory dir = getDirectory(createTempDir("testShorts")); |
| IndexOutput output = dir.createOutput("shorts", newIOContext(random())); |
| int num = TestUtil.nextInt(random(), 50, 3000); |
| short shorts[] = new short[num]; |
| for (int i = 0; i < shorts.length; i++) { |
| shorts[i] = (short) random().nextInt(); |
| output.writeShort(shorts[i]); |
| } |
| output.close(); |
| |
| // slice |
| IndexInput input = dir.openInput("shorts", newIOContext(random())); |
| RandomAccessInput slice = input.randomAccessSlice(0, input.length()); |
| for (int i = 0; i < shorts.length; i++) { |
| assertEquals(shorts[i], slice.readShort(i * 2)); |
| } |
| |
| // subslices |
| for (int i = 1; i < shorts.length; i++) { |
| long offset = i * 2; |
| RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); |
| for (int j = i; j < shorts.length; j++) { |
| assertEquals(shorts[j], subslice.readShort((j - i) * 2)); |
| } |
| } |
| |
| // with padding |
| for (int i = 0; i < 7; i++) { |
| String name = "shorts-" + i; |
| IndexOutput o = dir.createOutput(name, newIOContext(random())); |
| byte junk[] = new byte[i]; |
| random().nextBytes(junk); |
| o.writeBytes(junk, junk.length); |
| input.seek(0); |
| o.copyBytes(input, input.length()); |
| o.close(); |
| IndexInput padded = dir.openInput(name, newIOContext(random())); |
| RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); |
| for (int j = 0; j < shorts.length; j++) { |
| assertEquals(shorts[j], whole.readShort(j * 2)); |
| } |
| padded.close(); |
| } |
| input.close(); |
| dir.close(); |
| } |
| |
| public void testRandomByte() throws Exception { |
| Directory dir = getDirectory(createTempDir("testBytes")); |
| IndexOutput output = dir.createOutput("bytes", newIOContext(random())); |
| int num = TestUtil.nextInt(random(), 50, 3000); |
| byte bytes[] = new byte[num]; |
| random().nextBytes(bytes); |
| for (int i = 0; i < bytes.length; i++) { |
| output.writeByte(bytes[i]); |
| } |
| output.close(); |
| |
| // slice |
| IndexInput input = dir.openInput("bytes", newIOContext(random())); |
| RandomAccessInput slice = input.randomAccessSlice(0, input.length()); |
| for (int i = 0; i < bytes.length; i++) { |
| assertEquals(bytes[i], slice.readByte(i)); |
| } |
| |
| // subslices |
| for (int i = 1; i < bytes.length; i++) { |
| long offset = i; |
| RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); |
| for (int j = i; j < bytes.length; j++) { |
| assertEquals(bytes[j], subslice.readByte(j - i)); |
| } |
| } |
| |
| // with padding |
| for (int i = 0; i < 7; i++) { |
| String name = "bytes-" + i; |
| IndexOutput o = dir.createOutput(name, newIOContext(random())); |
| byte junk[] = new byte[i]; |
| random().nextBytes(junk); |
| o.writeBytes(junk, junk.length); |
| input.seek(0); |
| o.copyBytes(input, input.length()); |
| o.close(); |
| IndexInput padded = dir.openInput(name, newIOContext(random())); |
| RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); |
| for (int j = 0; j < bytes.length; j++) { |
| assertEquals(bytes[j], whole.readByte(j)); |
| } |
| padded.close(); |
| } |
| input.close(); |
| dir.close(); |
| } |
| |
| /** try to stress slices of slices */ |
| public void testSliceOfSlice() throws Exception { |
| Directory dir = getDirectory(createTempDir("sliceOfSlice")); |
| IndexOutput output = dir.createOutput("bytes", newIOContext(random())); |
| final int num; |
| if (TEST_NIGHTLY) { |
| num = TestUtil.nextInt(random(), 250, 2500); |
| } else { |
| num = TestUtil.nextInt(random(), 50, 250); |
| } |
| byte bytes[] = new byte[num]; |
| random().nextBytes(bytes); |
| for (int i = 0; i < bytes.length; i++) { |
| output.writeByte(bytes[i]); |
| } |
| output.close(); |
| |
| IndexInput input = dir.openInput("bytes", newIOContext(random())); |
| // seek to a random spot shouldnt impact slicing. |
| input.seek(TestUtil.nextLong(random(), 0, input.length())); |
| for (int i = 0; i < num; i += 16) { |
| IndexInput slice1 = input.slice("slice1", i, num-i); |
| assertEquals(0, slice1.getFilePointer()); |
| assertEquals(num-i, slice1.length()); |
| |
| // seek to a random spot shouldnt impact slicing. |
| slice1.seek(TestUtil.nextLong(random(), 0, slice1.length())); |
| for (int j = 0; j < slice1.length(); j += 16) { |
| IndexInput slice2 = slice1.slice("slice2", j, num-i-j); |
| assertEquals(0, slice2.getFilePointer()); |
| assertEquals(num-i-j, slice2.length()); |
| byte data[] = new byte[num]; |
| System.arraycopy(bytes, 0, data, 0, i+j); |
| if (random().nextBoolean()) { |
| // read the bytes for this slice-of-slice |
| slice2.readBytes(data, i+j, num-i-j); |
| } else { |
| // seek to a random spot in between, read some, seek back and read the rest |
| long seek = TestUtil.nextLong(random(), 0, slice2.length()); |
| slice2.seek(seek); |
| slice2.readBytes(data, (int)(i+j+seek), (int)(num-i-j-seek)); |
| slice2.seek(0); |
| slice2.readBytes(data, i+j, (int)seek); |
| } |
| assertArrayEquals(bytes, data); |
| } |
| } |
| |
| input.close(); |
| dir.close(); |
| } |
| |
| /** |
| * This test that writes larger than the size of the buffer output |
| * will correctly increment the file pointer. |
| */ |
| public void testLargeWrites() throws IOException { |
| Directory dir = getDirectory(createTempDir("largeWrites")); |
| IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random())); |
| |
| byte[] largeBuf = new byte[2048]; |
| random().nextBytes(largeBuf); |
| |
| long currentPos = os.getFilePointer(); |
| os.writeBytes(largeBuf, largeBuf.length); |
| |
| try { |
| assertEquals(currentPos + largeBuf.length, os.getFilePointer()); |
| } finally { |
| os.close(); |
| } |
| dir.close(); |
| } |
| |
| // LUCENE-6084 |
| public void testIndexOutputToString() throws Throwable { |
| Directory dir = getDirectory(createTempDir()); |
| IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random())); |
| assertTrue(out.toString(), out.toString().contains("camelCase.txt")); |
| out.close(); |
| dir.close(); |
| } |
| |
| public void testDoubleCloseDirectory() throws Throwable { |
| Directory dir = getDirectory(createTempDir()); |
| IndexOutput out = dir.createOutput("foobar", newIOContext(random())); |
| out.writeString("testing"); |
| out.close(); |
| dir.close(); |
| dir.close(); // close again |
| } |
| |
| public void testDoubleCloseOutput() throws Throwable { |
| Directory dir = getDirectory(createTempDir()); |
| IndexOutput out = dir.createOutput("foobar", newIOContext(random())); |
| out.writeString("testing"); |
| out.close(); |
| out.close(); // close again |
| dir.close(); |
| } |
| |
| public void testDoubleCloseInput() throws Throwable { |
| Directory dir = getDirectory(createTempDir()); |
| IndexOutput out = dir.createOutput("foobar", newIOContext(random())); |
| out.writeString("testing"); |
| out.close(); |
| IndexInput in = dir.openInput("foobar", newIOContext(random())); |
| assertEquals("testing", in.readString()); |
| in.close(); |
| in.close(); // close again |
| dir.close(); |
| } |
| |
| public void testCreateTempOutput() throws Throwable { |
| Directory dir = getDirectory(createTempDir()); |
| List<String> names = new ArrayList<>(); |
| int iters = atLeast(50); |
| for(int iter=0;iter<iters;iter++) { |
| IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random())); |
| names.add(out.getName()); |
| out.writeVInt(iter); |
| out.close(); |
| } |
| for(int iter=0;iter<iters;iter++) { |
| IndexInput in = dir.openInput(names.get(iter), newIOContext(random())); |
| assertEquals(iter, in.readVInt()); |
| in.close(); |
| } |
| Set<String> files = new HashSet<String>(Arrays.asList(dir.listAll())); |
| // In case ExtrasFS struck: |
| files.remove("extra0"); |
| assertEquals(new HashSet<String>(names), files); |
| dir.close(); |
| } |
| } |
