| Index: lucene/misc/src/test/org/apache/lucene/store/TestNativeUnixDirectory.java |
| =================================================================== |
| --- lucene/misc/src/test/org/apache/lucene/store/TestNativeUnixDirectory.java (revision 0) |
| +++ lucene/misc/src/test/org/apache/lucene/store/TestNativeUnixDirectory.java (working copy) |
| @@ -0,0 +1,36 @@ |
| +package org.apache.lucene.store; |
| + |
| +/* |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| +import java.io.File; |
| +import java.io.IOException; |
| + |
| +import org.apache.lucene.util.Constants; |
| +import org.apache.lucene.util.TestUtil; |
| +import org.junit.Assume; |
| + |
| +public class TestNativeUnixDirectory extends BaseDirectoryTestCase { |
| + |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + // NativeUnixDirectory only works on Unix: |
| + Assume.assumeTrue(Constants.WINDOWS == false); |
| + |
| + return new NativeUnixDirectory(FSDirectory.open(path), TestUtil.nextInt(random(), 1, 1000)*512, random().nextInt(10000), true); |
| + } |
| +} |
| |
| Property changes on: lucene/misc/src/test/org/apache/lucene/store/TestNativeUnixDirectory.java |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| ## -0,0 +1 ## |
| +native |
| \ No newline at end of property |
| Index: lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java |
| =================================================================== |
| --- lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (revision 1583585) |
| +++ lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (working copy) |
| @@ -19,15 +19,17 @@ |
| |
| import java.io.EOFException; |
| import java.io.File; |
| -import java.io.IOException; |
| +import java.io.FileDescriptor; |
| import java.io.FileInputStream; |
| -import java.io.FileDescriptor; |
| import java.io.FileOutputStream; |
| +import java.io.IOException; |
| import java.nio.ByteBuffer; |
| import java.nio.channels.FileChannel; |
| +import java.util.zip.CRC32; |
| |
| import org.apache.lucene.store.Directory; // javadoc |
| import org.apache.lucene.store.IOContext.Context; |
| +import org.apache.lucene.util.IOUtils; |
| |
| // TODO |
| // - newer Linux kernel versions (after 2.6.29) have |
| @@ -36,6 +38,15 @@ |
| // cache; we should explore using that instead of direct |
| // IO when context is merge |
| |
| +// NOTE: to run tests using this Dir: |
| +// edit the ctor taking File to make the minBytesDirect |
| +// smaller, because very few (no?) tests will actually do |
| +// merges > 10 MB so O_DIRECT won't be used |
| +// cd lucene/misc |
| +// ant build-native-unix jar |
| +// |
| +// ant test -lib /path/to/lucene/build/misc/lucene-misc-5.0-SNAPSHOT.jar -Dtests.directory=NativeUnixDirectory |
| + |
| /** |
| * A {@link Directory} implementation for all Unixes that uses |
| * DIRECT I/O to bypass OS level IO caching during |
| @@ -67,7 +78,7 @@ |
| * |
| * @lucene.experimental |
| */ |
| -public class NativeUnixDirectory extends FSDirectory { |
| +public class NativeUnixDirectory extends FilterDirectory { |
| |
| // TODO: this is OS dependent, but likely 512 is the LCD |
| private final static long ALIGN = 512; |
| @@ -86,46 +97,54 @@ |
| private final int mergeBufferSize; |
| private final long minBytesDirect; |
| private final Directory delegate; |
| + private final boolean doCloseDelegate; |
| |
| - /** Create a new NIOFSDirectory for the named location. |
| + /** Create a new NativeUnixDirectory for the named location. |
| * |
| - * @param path the path of the directory |
| + * @param delegate the wrapped directory, used for flushes |
| + * or small merges |
| * @param mergeBufferSize Size of buffer to use for |
| * merging. See {@link #DEFAULT_MERGE_BUFFER_SIZE}. |
| * @param minBytesDirect Merges, or files to be opened for |
| * reading, smaller than this will |
| * not use direct IO. See {@link |
| * #DEFAULT_MIN_BYTES_DIRECT} |
| - * @param delegate fallback Directory for non-merges |
| * @throws IOException If there is a low-level I/O error |
| */ |
| - public NativeUnixDirectory(File path, int mergeBufferSize, long minBytesDirect, Directory delegate) throws IOException { |
| - super(path, delegate.getLockFactory()); |
| - if ((mergeBufferSize & ALIGN) != 0) { |
| + public NativeUnixDirectory(FSDirectory delegate, int mergeBufferSize, long minBytesDirect, boolean doCloseDelegate) throws IOException { |
| + super(delegate); |
| + if ((mergeBufferSize & (ALIGN-1)) != 0) { |
| throw new IllegalArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")"); |
| } |
| this.mergeBufferSize = mergeBufferSize; |
| this.minBytesDirect = minBytesDirect; |
| this.delegate = delegate; |
| + this.doCloseDelegate = doCloseDelegate; |
| } |
| |
| - /** Create a new NIOFSDirectory for the named location. |
| + /** Create a new NativeUnixDirectory for the named location. |
| * |
| - * @param path the path of the directory |
| - * @param delegate fallback Directory for non-merges |
| + * @param delegate the wrapped directory, used for flushes |
| + * or small merges |
| * @throws IOException If there is a low-level I/O error |
| */ |
| - public NativeUnixDirectory(File path, Directory delegate) throws IOException { |
| - this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, delegate); |
| + public NativeUnixDirectory(FSDirectory delegate, boolean doCloseDelegate) throws IOException { |
| + this(delegate, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, doCloseDelegate); |
| } |
| |
| + /** Create a new NativeUnixDirectory wrapping {@link |
| + * FSDirectory#open}. */ |
| + public NativeUnixDirectory(File path) throws IOException { |
| + this(FSDirectory.open(path), DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, true); |
| + } |
| + |
| @Override |
| public IndexInput openInput(String name, IOContext context) throws IOException { |
| ensureOpen(); |
| - if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect || fileLength(name) < minBytesDirect) { |
| + if (context.context != Context.MERGE || fileLength(name) < minBytesDirect) { |
| return delegate.openInput(name, context); |
| } else { |
| - return new NativeUnixIndexInput(new File(getDirectory(), name), mergeBufferSize); |
| + return new NativeUnixIndexInput(new File(((FSDirectory) in).getDirectory(), name), mergeBufferSize); |
| } |
| } |
| |
| @@ -135,16 +154,21 @@ |
| if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect) { |
| return delegate.createOutput(name, context); |
| } else { |
| - ensureCanWrite(name); |
| - return new NativeUnixIndexOutput(new File(getDirectory(), name), mergeBufferSize); |
| + ((FSDirectory) in).ensureCanWrite(name); |
| + return new NativeUnixIndexOutput(new File(((FSDirectory) in).getDirectory(), name), mergeBufferSize); |
| } |
| } |
| |
| + // NOTE: we can't just subclass BufferedIndexOutput |
| + // becaues it sometimes flushes partial buffers not at the |
| + // end of the file (e.g. in writeBytes): |
| + |
| private final static class NativeUnixIndexOutput extends IndexOutput { |
| private final ByteBuffer buffer; |
| private final FileOutputStream fos; |
| private final FileChannel channel; |
| private final int bufferSize; |
| + private final CRC32 crc = new CRC32(); |
| |
| //private final File path; |
| |
| @@ -155,11 +179,17 @@ |
| |
| public NativeUnixIndexOutput(File path, int bufferSize) throws IOException { |
| //this.path = path; |
| + // TODO: we can reflect into |
| + // sun.nio.fs.UnixNativeDispatcher's open method |
| + // instead of our own: |
| final FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), false); |
| fos = new FileOutputStream(fd); |
| - //fos = new FileOutputStream(path); |
| + //System.out.println("FD=" + fd + " path=" + path); |
| channel = fos.getChannel(); |
| - buffer = ByteBuffer.allocateDirect(bufferSize); |
| + // TODO: we can use Unsafe to get page size alignment, |
| + // and then alloc a buffer and do our own custom align |
| + // within that? |
| + buffer = NativePosixUtil.newAlignedDirectByteBuffer(ALIGN, bufferSize); |
| this.bufferSize = bufferSize; |
| isOpen = true; |
| } |
| @@ -168,6 +198,7 @@ |
| public void writeByte(byte b) throws IOException { |
| assert bufferPos == buffer.position(): "bufferPos=" + bufferPos + " vs buffer.position()=" + buffer.position(); |
| buffer.put(b); |
| + crc.update(b); |
| if (++bufferPos == bufferSize) { |
| dump(); |
| } |
| @@ -175,6 +206,7 @@ |
| |
| @Override |
| public void writeBytes(byte[] src, int offset, int len) throws IOException { |
| + crc.update(src, offset, len); |
| int toWrite = len; |
| while(true) { |
| final int left = bufferSize - bufferPos; |
| @@ -243,7 +275,7 @@ |
| |
| @Override |
| public long getChecksum() throws IOException { |
| - throw new UnsupportedOperationException("this directory currently does not work at all!"); |
| + return crc.getValue(); |
| } |
| |
| @Override |
| @@ -254,6 +286,9 @@ |
| dump(); |
| } finally { |
| try { |
| + // Sets the fileLength to the true length; this |
| + // is necessary because we rounded up to the |
| + // next multiple of bufferSize when writing: |
| //System.out.println("direct close set len=" + fileLength + " vs " + channel.size() + " path=" + path); |
| channel.truncate(fileLength); |
| //System.out.println(" now: " + channel.size()); |
| @@ -283,11 +318,17 @@ |
| |
| public NativeUnixIndexInput(File path, int bufferSize) throws IOException { |
| super("NativeUnixIndexInput(path=\"" + path.getPath() + "\")"); |
| + // TODO: we can reflect into |
| + // sun.nio.fs.UnixNativeDispatcher's open method |
| + // instead of our own: |
| final FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true); |
| fis = new FileInputStream(fd); |
| channel = fis.getChannel(); |
| this.bufferSize = bufferSize; |
| - buffer = ByteBuffer.allocateDirect(bufferSize); |
| + // TODO: we can use Unsafe to get page size alignment, |
| + // and then alloc a buffer and do our own custom align |
| + // within that? |
| + buffer = NativePosixUtil.newAlignedDirectByteBuffer(ALIGN, bufferSize); |
| isOpen = true; |
| isClone = false; |
| filePos = -bufferSize; |
| @@ -301,7 +342,7 @@ |
| this.fis = null; |
| channel = other.channel; |
| this.bufferSize = other.bufferSize; |
| - buffer = ByteBuffer.allocateDirect(bufferSize); |
| + buffer = NativePosixUtil.newAlignedDirectByteBuffer(ALIGN, bufferSize); |
| filePos = -bufferSize; |
| bufferPos = bufferSize; |
| isOpen = true; |
| @@ -417,4 +458,15 @@ |
| } |
| } |
| } |
| + |
| + @Override |
| + public void close() throws IOException { |
| + try { |
| + super.close(); |
| + } finally { |
| + if (doCloseDelegate) { |
| + delegate.close(); |
| + } |
| + } |
| + } |
| } |
| Index: lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java |
| =================================================================== |
| --- lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java (revision 1583585) |
| +++ lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.java (working copy) |
| @@ -41,6 +41,7 @@ |
| public static native int posix_madvise(ByteBuffer buf, int advise) throws IOException; |
| public static native int madvise(ByteBuffer buf, int advise) throws IOException; |
| public static native FileDescriptor open_direct(String filename, boolean read) throws IOException; |
| + public static native ByteBuffer newAlignedDirectByteBuffer(long alignment, long size) throws IOException; |
| public static native long pread(FileDescriptor fd, long pos, ByteBuffer byteBuf) throws IOException; |
| |
| public static void advise(FileDescriptor fd, long offset, long len, int advise) throws IOException { |
| Index: lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp |
| =================================================================== |
| --- lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp (revision 1583585) |
| +++ lucene/misc/src/java/org/apache/lucene/store/NativePosixUtil.cpp (working copy) |
| @@ -30,6 +30,7 @@ |
| #include <string.h> // strerror |
| #include <errno.h> // errno |
| #include <unistd.h> // pread |
| +#include <stdlib.h> // posix_malign |
| #include <sys/mman.h> // posix_madvise, madvise |
| #include <sys/types.h> // constants for open |
| #include <sys/stat.h> // constants for open |
| @@ -112,15 +113,11 @@ |
| { |
| jfieldID field_fd; |
| jmethodID const_fdesc; |
| - jclass class_fdesc, class_ioex; |
| + jclass class_fdesc; |
| jobject ret; |
| int fd; |
| char *fname; |
| |
| - class_ioex = env->FindClass("java/io/IOException"); |
| - if (class_ioex == NULL) { |
| - return NULL; |
| - } |
| class_fdesc = env->FindClass("java/io/FileDescriptor"); |
| if (class_fdesc == NULL) { |
| return NULL; |
| @@ -136,7 +133,7 @@ |
| } else { |
| fd = open(fname, O_RDWR | O_CREAT | DIRECT_FLAG, 0666); |
| #ifdef OSX |
| - fcntl(fd, F_NOCACHE, 1); |
| + fcntl(fd, F_NOCACHE, 1); |
| #endif |
| } |
| |
| @@ -145,8 +142,18 @@ |
| env->ReleaseStringUTFChars(filename, fname); |
| |
| if (fd < 0) { |
| - // open returned an error. Throw an IOException with the error string |
| - env->ThrowNew(class_ioex, strerror(errno)); |
| + if (errno == ENOENT) { |
| + jclass class_fnfeex = env->FindClass("java/io/FileNotFoundException"); |
| + if (class_fnfeex != NULL) { |
| + env->ThrowNew(class_fnfeex, strerror(errno)); |
| + } |
| + } else { |
| + // open returned an error. Throw an IOException with the error string |
| + jclass class_ioex = env->FindClass("java/io/IOException"); |
| + if (class_ioex != NULL) { |
| + env->ThrowNew(class_ioex, strerror(errno)); |
| + } |
| + } |
| return NULL; |
| } |
| |
| @@ -168,6 +175,20 @@ |
| return ret; |
| } |
| |
| +extern "C" |
| +JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_newAlignedDirectByteBuffer(JNIEnv *env, jclass _ignore, jlong alignment, jlong size) { |
| + void *p; |
| + int result = posix_memalign(&p, (size_t) alignment, (size_t) size); |
| + if (result != 0) { |
| + jclass class_ioex = env->FindClass("java/io/IOException"); |
| + if (class_ioex != NULL) { |
| + env->ThrowNew(class_ioex, strerror(result)); |
| + } |
| + return NULL; |
| + } |
| + return env->NewDirectByteBuffer(p, size); |
| +} |
| + |
| /* |
| * Class: org_apache_lucene_store_NativePosixUtil |
| * Method: pread |
| Index: lucene/misc/build.xml |
| =================================================================== |
| --- lucene/misc/build.xml (revision 1583585) |
| +++ lucene/misc/build.xml (working copy) |
| @@ -41,7 +41,11 @@ |
| <taskdef resource="cpptasks.tasks" classpathref="cpptasks.classpath"/> |
| </target> |
| |
| - <target name="build-native-unix" depends="install-cpptasks"> |
| + <condition property="isUnix"> |
| + <os family="unix"/> |
| + </condition> |
| + |
| + <target name="build-native-unix" depends="install-cpptasks" if="isUnix"> |
| <mkdir dir="${common.build.dir}/native"/> |
| |
| <cc outtype="shared" name="c++" subsystem="console" outfile="${common.build.dir}/native/NativePosixUtil" > |
| @@ -58,4 +62,5 @@ |
| </cc> |
| </target> |
| |
| + <target name="init" depends="build-native-unix,module-build.init"/> |
| </project> |
| Index: lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java |
| =================================================================== |
| --- lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (revision 0) |
| +++ lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (working copy) |
| @@ -0,0 +1,400 @@ |
| +package org.apache.lucene.store; |
| + |
| +/* |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| +import java.io.File; |
| +import java.io.FileNotFoundException; |
| +import java.io.IOException; |
| +import java.nio.file.NoSuchFileException; |
| +import java.util.Arrays; |
| + |
| +import org.apache.lucene.index.DirectoryReader; |
| +import org.apache.lucene.index.IndexNotFoundException; |
| +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; |
| +import org.apache.lucene.util.IOUtils; |
| +import org.apache.lucene.util.LuceneTestCase; |
| +import org.apache.lucene.util.TestUtil; |
| + |
| +/** Base class for per-Directory tests. */ |
| + |
| +public abstract class BaseDirectoryTestCase extends LuceneTestCase { |
| + |
| + /** Subclass returns the Directory to be tested; if it's |
| + * an FS-based directory it should point to the specified |
| + * path, else it can ignore it. */ |
| + protected abstract Directory getDirectory(File path) throws IOException; |
| + |
| + /** Make sure directory throws AlreadyClosedException if |
| + * you try to createOutput after closing. */ |
| + public void testDetectClose() throws Throwable { |
| + Directory dir = getDirectory(TestUtil.getTempDir("testDetectClose")); |
| + dir.close(); |
| + try { |
| + dir.createOutput("test", newIOContext(random())); |
| + fail("did not hit expected exception"); |
| + } catch (AlreadyClosedException ace) { |
| + // expected |
| + } |
| + } |
| + |
| + // test is occasionally very slow, i dont know why |
| + // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE |
| + @Nightly |
| + public void testThreadSafety() throws Exception { |
| + final BaseDirectoryWrapper dir = newDirectory(getDirectory(TestUtil.getTempDir("testThreadSafety"))); |
| + dir.setCheckIndexOnClose(false); // we arent making an index |
| + if (dir instanceof MockDirectoryWrapper) { |
| + ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); // makes this test really slow |
| + } |
| + |
| + if (VERBOSE) { |
| + System.out.println(dir); |
| + } |
| + |
| + class TheThread extends Thread { |
| + private String name; |
| + |
| + public TheThread(String name) { |
| + this.name = name; |
| + } |
| + |
| + @Override |
| + public void run() { |
| + for (int i = 0; i < 3000; i++) { |
| + String fileName = this.name + i; |
| + try { |
| + //System.out.println("create:" + fileName); |
| + IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| + output.close(); |
| + assertTrue(slowFileExists(dir, fileName)); |
| + } catch (IOException e) { |
| + throw new RuntimeException(e); |
| + } |
| + } |
| + } |
| + }; |
| + |
| + class TheThread2 extends Thread { |
| + private String name; |
| + |
| + public TheThread2(String name) { |
| + this.name = name; |
| + } |
| + |
| + @Override |
| + public void run() { |
| + for (int i = 0; i < 10000; i++) { |
| + try { |
| + String[] files = dir.listAll(); |
| + for (String file : files) { |
| + //System.out.println("file:" + file); |
| + try { |
| + IndexInput input = dir.openInput(file, newIOContext(random())); |
| + input.close(); |
| + } catch (FileNotFoundException | NoSuchFileException e) { |
| + // ignore |
| + } catch (IOException e) { |
| + if (e.getMessage().contains("still open for writing")) { |
| + // ignore |
| + } else { |
| + throw new RuntimeException(e); |
| + } |
| + } |
| + if (random().nextBoolean()) { |
| + break; |
| + } |
| + } |
| + } catch (IOException e) { |
| + throw new RuntimeException(e); |
| + } |
| + } |
| + } |
| + }; |
| + |
| + TheThread theThread = new TheThread("t1"); |
| + TheThread2 theThread2 = new TheThread2("t2"); |
| + theThread.start(); |
| + theThread2.start(); |
| + |
| + theThread.join(); |
| + theThread2.join(); |
| + |
| + dir.close(); |
| + } |
| + |
| + /** LUCENE-1464: just creating a Directory should not |
| + * mkdir the underling directory in the filesystem. */ |
| + public void testDontCreate() throws Throwable { |
| + File path = new File(TEMP_DIR, "doesnotexist"); |
| + try { |
| + assertTrue(!path.exists()); |
| + Directory dir = getDirectory(path); |
| + assertTrue(!path.exists()); |
| + dir.close(); |
| + } finally { |
| + TestUtil.rmDir(path); |
| + } |
| + } |
| + |
| + /** LUCENE-1468: once we create an output, we should see |
| + * it in the dir listing and be able to open it with |
| + * openInput. */ |
| + public void testDirectoryFilter() throws IOException { |
| + String name = "file"; |
| + Directory dir = getDirectory(TestUtil.getTempDir("testDirectoryFilter")); |
| + try { |
| + dir.createOutput(name, newIOContext(random())).close(); |
| + assertTrue(slowFileExists(dir, name)); |
| + assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| + } finally { |
| + dir.close(); |
| + } |
| + } |
| + |
| + // LUCENE-2852 |
| + public void testSeekToEOFThenBack() throws Exception { |
| + Directory dir = getDirectory(TestUtil.getTempDir("testSeekToEOFThenBack")); |
| + |
| + IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| + byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE]; |
| + o.writeBytes(bytes, 0, bytes.length); |
| + o.close(); |
| + |
| + IndexInput i = dir.openInput("out", newIOContext(random())); |
| + i.seek(2*RAMInputStream.BUFFER_SIZE-1); |
| + i.seek(3*RAMInputStream.BUFFER_SIZE); |
| + i.seek(RAMInputStream.BUFFER_SIZE); |
| + i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE); |
| + i.close(); |
| + dir.close(); |
| + } |
| + |
| + // LUCENE-1196 |
| + public void testIllegalEOF() throws Exception { |
| + Directory dir = getDirectory(TestUtil.getTempDir("testIllegalEOF")); |
| + IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| + byte[] b = new byte[1024]; |
| + o.writeBytes(b, 0, 1024); |
| + o.close(); |
| + IndexInput i = dir.openInput("out", newIOContext(random())); |
| + i.seek(1024); |
| + i.close(); |
| + dir.close(); |
| + } |
| + |
| + public void testDeleteFile() throws Exception { |
| + Directory dir = getDirectory(TestUtil.getTempDir("testDeleteFile")); |
| + dir.createOutput("foo.txt", IOContext.DEFAULT).close(); |
| + dir.deleteFile("foo.txt"); |
| + assertEquals(0, dir.listAll().length); |
| + dir.close(); |
| + } |
| + |
| + // LUCENE-3382 -- make sure we get exception if the directory really does not exist. |
| + public void testNoDir() throws Throwable { |
| + File tempDir = TestUtil.getTempDir("doesnotexist"); |
| + TestUtil.rmDir(tempDir); |
| + Directory dir = getDirectory(tempDir); |
| + try { |
| + DirectoryReader.open(dir); |
| + fail("did not hit expected exception"); |
| + } catch (NoSuchDirectoryException | IndexNotFoundException nsde) { |
| + // expected |
| + } |
| + dir.close(); |
| + } |
| + |
| + // LUCENE-3382 test that delegate compound files correctly. |
| + public void testCompoundFileAppendTwice() throws IOException { |
| + Directory newDir = getDirectory(TestUtil.getTempDir("testCompoundFileAppendTwice")); |
| + CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true); |
| + createSequenceFile(newDir, "d1", (byte) 0, 15); |
| + IndexOutput out = csw.createOutput("d.xyz", newIOContext(random())); |
| + out.writeInt(0); |
| + out.close(); |
| + assertEquals(1, csw.listAll().length); |
| + assertEquals("d.xyz", csw.listAll()[0]); |
| + |
| + csw.close(); |
| + |
| + CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false); |
| + assertEquals(1, cfr.listAll().length); |
| + assertEquals("d.xyz", cfr.listAll()[0]); |
| + cfr.close(); |
| + newDir.close(); |
| + } |
| + |
| + |
| + /** Creates a file of the specified size with sequential data. The first |
| + * byte is written as the start byte provided. All subsequent bytes are |
| + * computed as start + offset where offset is the number of the byte. |
| + */ |
| + private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException { |
| + IndexOutput os = dir.createOutput(name, newIOContext(random())); |
| + for (int i=0; i < size; i++) { |
| + os.writeByte(start); |
| + start ++; |
| + } |
| + os.close(); |
| + } |
| + |
| + public void testCopyBytes() throws Exception { |
| + testCopyBytes(getDirectory(TestUtil.getTempDir("testCopyBytes"))); |
| + } |
| + |
| + private static byte value(int idx) { |
| + return (byte) ((idx % 256) * (1 + (idx / 256))); |
| + } |
| + |
| + public static void testCopyBytes(Directory dir) throws Exception { |
| + |
| + // make random file |
| + IndexOutput out = dir.createOutput("test", newIOContext(random())); |
| + byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)]; |
| + final int size = TestUtil.nextInt(random(), 1, 1777777); |
| + int upto = 0; |
| + int byteUpto = 0; |
| + while (upto < size) { |
| + bytes[byteUpto++] = value(upto); |
| + upto++; |
| + if (byteUpto == bytes.length) { |
| + out.writeBytes(bytes, 0, bytes.length); |
| + byteUpto = 0; |
| + } |
| + } |
| + |
| + out.writeBytes(bytes, 0, byteUpto); |
| + assertEquals(size, out.getFilePointer()); |
| + out.close(); |
| + assertEquals(size, dir.fileLength("test")); |
| + |
| + // copy from test -> test2 |
| + final IndexInput in = dir.openInput("test", newIOContext(random())); |
| + |
| + out = dir.createOutput("test2", newIOContext(random())); |
| + |
| + upto = 0; |
| + while (upto < size) { |
| + if (random().nextBoolean()) { |
| + out.writeByte(in.readByte()); |
| + upto++; |
| + } else { |
| + final int chunk = Math.min( |
| + TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| + out.copyBytes(in, chunk); |
| + upto += chunk; |
| + } |
| + } |
| + assertEquals(size, upto); |
| + out.close(); |
| + in.close(); |
| + |
| + // verify |
| + IndexInput in2 = dir.openInput("test2", newIOContext(random())); |
| + upto = 0; |
| + while (upto < size) { |
| + if (random().nextBoolean()) { |
| + final byte v = in2.readByte(); |
| + assertEquals(value(upto), v); |
| + upto++; |
| + } else { |
| + final int limit = Math.min( |
| + TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| + in2.readBytes(bytes, 0, limit); |
| + for (int byteIdx = 0; byteIdx < limit; byteIdx++) { |
| + assertEquals(value(upto), bytes[byteIdx]); |
| + upto++; |
| + } |
| + } |
| + } |
| + in2.close(); |
| + |
| + dir.deleteFile("test"); |
| + dir.deleteFile("test2"); |
| + |
| + dir.close(); |
| + } |
| + |
| + // LUCENE-3541 |
| + public void testCopyBytesWithThreads() throws Exception { |
| + testCopyBytesWithThreads(getDirectory(TestUtil.getTempDir("testCopyBytesWithThreads"))); |
| + } |
| + |
| + public static void testCopyBytesWithThreads(Directory d) throws Exception { |
| + int datalen = TestUtil.nextInt(random(), 101, 10000); |
| + byte data[] = new byte[datalen]; |
| + random().nextBytes(data); |
| + |
| + IndexOutput output = d.createOutput("data", IOContext.DEFAULT); |
| + output.writeBytes(data, 0, datalen); |
| + output.close(); |
| + |
| + IndexInput input = d.openInput("data", IOContext.DEFAULT); |
| + IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT); |
| + // copy our 100-byte header |
| + outputHeader.copyBytes(input, 100); |
| + outputHeader.close(); |
| + |
| + // now make N copies of the remaining bytes |
| + CopyThread copies[] = new CopyThread[10]; |
| + for (int i = 0; i < copies.length; i++) { |
| + copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT)); |
| + } |
| + |
| + for (int i = 0; i < copies.length; i++) { |
| + copies[i].start(); |
| + } |
| + |
| + for (int i = 0; i < copies.length; i++) { |
| + copies[i].join(); |
| + } |
| + |
| + for (int i = 0; i < copies.length; i++) { |
| + IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT); |
| + byte[] dataCopy = new byte[datalen]; |
| + System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing |
| + copiedData.readBytes(dataCopy, 100, datalen-100); |
| + assertArrayEquals(data, dataCopy); |
| + copiedData.close(); |
| + } |
| + input.close(); |
| + d.close(); |
| + |
| + } |
| + |
| + static class CopyThread extends Thread { |
| + final IndexInput src; |
| + final IndexOutput dst; |
| + |
| + CopyThread(IndexInput src, IndexOutput dst) { |
| + this.src = src; |
| + this.dst = dst; |
| + } |
| + |
| + @Override |
| + public void run() { |
| + try { |
| + dst.copyBytes(src, src.length()-100); |
| + dst.close(); |
| + } catch (IOException ex) { |
| + throw new RuntimeException(ex); |
| + } |
| + } |
| + } |
| +} |
| + |
| |
| Property changes on: lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| ## -0,0 +1 ## |
| +native |
| \ No newline at end of property |
| Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java |
| =================================================================== |
| --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1583585) |
| +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy) |
| @@ -1170,6 +1170,21 @@ |
| return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir); |
| } |
| |
| + // See if it has a File ctor even though it's not an |
| + // FSDir subclass: |
| + Constructor<? extends Directory> fileCtor = null; |
| + try { |
| + fileCtor = clazz.getConstructor(File.class); |
| + } catch (NoSuchMethodException nsme) { |
| + // Ignore |
| + } |
| + |
| + if (fileCtor != null) { |
| + final File dir = TestUtil.getTempDir("index"); |
| + dir.mkdirs(); // ensure it's created so we 'have' it. |
| + return fileCtor.newInstance(dir); |
| + } |
| + |
| // try empty ctor |
| return clazz.newInstance(); |
| } catch (Exception e) { |
| @@ -1340,7 +1355,7 @@ |
| } |
| if (ex != null) { |
| if (VERBOSE) { |
| - System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads"); |
| + System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads"); |
| } |
| r.addReaderClosedListener(new ReaderClosedListener() { |
| @Override |
| Index: lucene/module-build.xml |
| =================================================================== |
| --- lucene/module-build.xml (revision 1583585) |
| +++ lucene/module-build.xml (working copy) |
| @@ -30,7 +30,7 @@ |
| |
| <!-- if you extend the classpath refid in one contrib's build.xml (add JARs), use this as basis: --> |
| <path id="base.classpath"> |
| - <pathelement location="${common.dir}/build/core/classes/java"/> |
| + <pathelement location="${common.dir}/build/core/classes/java"/> |
| </path> |
| |
| <!-- default classpath refid, can be overridden by contrib's build.xml (use the above base.classpath as basis): --> |
| Index: lucene/core/src/test/org/apache/lucene/store/TestNIOFSDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestNIOFSDirectory.java (working copy) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestNIOFSDirectory.java (working copy) |
| @@ -27,265 +27,11 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestDirectory extends LuceneTestCase { |
| - public void testDetectClose() throws Throwable { |
| - Directory[] dirs = new Directory[] { |
| - new RAMDirectory(), |
| - new SimpleFSDirectory(TEMP_DIR), |
| - new NIOFSDirectory(TEMP_DIR) |
| - }; |
| +public class TestNIOFSDirectory extends BaseDirectoryTestCase { |
| |
| - for (Directory dir : dirs) { |
| - dir.close(); |
| - try { |
| - dir.createOutput("test", newIOContext(random())); |
| - fail("did not hit expected exception"); |
| - } catch (AlreadyClosedException ace) { |
| - } |
| - } |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + return new NIOFSDirectory(path); |
| } |
| - |
| - // test is occasionally very slow, i dont know why |
| - // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE |
| - @Nightly |
| - public void testThreadSafety() throws Exception { |
| - final BaseDirectoryWrapper dir = newDirectory(); |
| - dir.setCheckIndexOnClose(false); // we arent making an index |
| - if (dir instanceof MockDirectoryWrapper) { |
| - ((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow |
| - } |
| - |
| - if (VERBOSE) { |
| - System.out.println(dir); |
| - } |
| - |
| - class TheThread extends Thread { |
| - private String name; |
| - |
| - public TheThread(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 3000; i++) { |
| - String fileName = this.name + i; |
| - try { |
| - //System.out.println("create:" + fileName); |
| - IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| - output.close(); |
| - assertTrue(slowFileExists(dir, fileName)); |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - class TheThread2 extends Thread { |
| - private String name; |
| - |
| - public TheThread2(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 10000; i++) { |
| - try { |
| - String[] files = dir.listAll(); |
| - for (String file : files) { |
| - //System.out.println("file:" + file); |
| - try { |
| - IndexInput input = dir.openInput(file, newIOContext(random())); |
| - input.close(); |
| - } catch (FileNotFoundException | NoSuchFileException e) { |
| - // ignore |
| - } catch (IOException e) { |
| - if (e.getMessage().contains("still open for writing")) { |
| - // ignore |
| - } else { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - if (random().nextBoolean()) { |
| - break; |
| - } |
| - } |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - TheThread theThread = new TheThread("t1"); |
| - TheThread2 theThread2 = new TheThread2("t2"); |
| - theThread.start(); |
| - theThread2.start(); |
| - |
| - theThread.join(); |
| - theThread2.join(); |
| - |
| - dir.close(); |
| - } |
| - |
| - |
| - // Test that different instances of FSDirectory can coexist on the same |
| - // path, can read, write, and lock files. |
| - public void testDirectInstantiation() throws Exception { |
| - final File path = TestUtil.getTempDir("testDirectInstantiation"); |
| - |
| - final byte[] largeBuffer = new byte[random().nextInt(256*1024)], largeReadBuffer = new byte[largeBuffer.length]; |
| - for (int i = 0; i < largeBuffer.length; i++) { |
| - largeBuffer[i] = (byte) i; // automatically loops with modulo |
| - } |
| - |
| - final FSDirectory[] dirs = new FSDirectory[] { |
| - new SimpleFSDirectory(path, null), |
| - new NIOFSDirectory(path, null), |
| - new MMapDirectory(path, null) |
| - }; |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - String fname = "foo." + i; |
| - String lockname = "foo" + i + ".lck"; |
| - IndexOutput out = dir.createOutput(fname, newIOContext(random())); |
| - out.writeByte((byte)i); |
| - out.writeBytes(largeBuffer, largeBuffer.length); |
| - out.close(); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - d2.ensureOpen(); |
| - assertTrue(slowFileExists(d2, fname)); |
| - assertEquals(1 + largeBuffer.length, d2.fileLength(fname)); |
| - |
| - // don't do read tests if unmapping is not supported! |
| - if (d2 instanceof MMapDirectory && !((MMapDirectory) d2).getUseUnmap()) |
| - continue; |
| - |
| - IndexInput input = d2.openInput(fname, newIOContext(random())); |
| - assertEquals((byte)i, input.readByte()); |
| - // read array with buffering enabled |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, true); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - // read again without using buffer |
| - input.seek(1L); |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, false); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - input.close(); |
| - } |
| - |
| - // delete with a different dir |
| - dirs[(i+1)%dirs.length].deleteFile(fname); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - assertFalse(slowFileExists(d2, fname)); |
| - } |
| - |
| - Lock lock = dir.makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - Lock lock2 = d2.makeLock(lockname); |
| - try { |
| - assertFalse(lock2.obtain(1)); |
| - } catch (LockObtainFailedException e) { |
| - // OK |
| - } |
| - } |
| - |
| - lock.close(); |
| - |
| - // now lock with different dir |
| - lock = dirs[(i+1)%dirs.length].makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - lock.close(); |
| - } |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - dir.close(); |
| - assertFalse(dir.isOpen); |
| - } |
| - |
| - TestUtil.rmDir(path); |
| - } |
| - |
| - // LUCENE-1464 |
| - public void testDontCreate() throws Throwable { |
| - File path = new File(TEMP_DIR, "doesnotexist"); |
| - try { |
| - assertTrue(!path.exists()); |
| - Directory dir = new SimpleFSDirectory(path, null); |
| - assertTrue(!path.exists()); |
| - dir.close(); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testRAMDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(new RAMDirectory()); |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testFSDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(newFSDirectory(TestUtil.getTempDir("test"))); |
| - } |
| - |
| - // LUCENE-1468 |
| - private void checkDirectoryFilter(Directory dir) throws IOException { |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testCopySubdir() throws Throwable { |
| - File path = TestUtil.getTempDir("testsubdir"); |
| - try { |
| - path.mkdirs(); |
| - new File(path, "subdir").mkdirs(); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testNotDirectory() throws Throwable { |
| - File path = TestUtil.getTempDir("testnotdir"); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - try { |
| - IndexOutput out = fsDir.createOutput("afile", newIOContext(random())); |
| - out.close(); |
| - assertTrue(slowFileExists(fsDir, "afile")); |
| - try { |
| - new SimpleFSDirectory(new File(path, "afile"), null); |
| - fail("did not hit expected exception"); |
| - } catch (NoSuchDirectoryException nsde) { |
| - // Expected |
| - } |
| - } finally { |
| - fsDir.close(); |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| } |
| |
| Index: lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java (working copy) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java (working copy) |
| @@ -27,265 +27,11 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestDirectory extends LuceneTestCase { |
| - public void testDetectClose() throws Throwable { |
| - Directory[] dirs = new Directory[] { |
| - new RAMDirectory(), |
| - new SimpleFSDirectory(TEMP_DIR), |
| - new NIOFSDirectory(TEMP_DIR) |
| - }; |
| +public class TestMMapDirectory extends BaseDirectoryTestCase { |
| |
| - for (Directory dir : dirs) { |
| - dir.close(); |
| - try { |
| - dir.createOutput("test", newIOContext(random())); |
| - fail("did not hit expected exception"); |
| - } catch (AlreadyClosedException ace) { |
| - } |
| - } |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + return new MMapDirectory(path); |
| } |
| - |
| - // test is occasionally very slow, i dont know why |
| - // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE |
| - @Nightly |
| - public void testThreadSafety() throws Exception { |
| - final BaseDirectoryWrapper dir = newDirectory(); |
| - dir.setCheckIndexOnClose(false); // we arent making an index |
| - if (dir instanceof MockDirectoryWrapper) { |
| - ((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow |
| - } |
| - |
| - if (VERBOSE) { |
| - System.out.println(dir); |
| - } |
| - |
| - class TheThread extends Thread { |
| - private String name; |
| - |
| - public TheThread(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 3000; i++) { |
| - String fileName = this.name + i; |
| - try { |
| - //System.out.println("create:" + fileName); |
| - IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| - output.close(); |
| - assertTrue(slowFileExists(dir, fileName)); |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - class TheThread2 extends Thread { |
| - private String name; |
| - |
| - public TheThread2(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 10000; i++) { |
| - try { |
| - String[] files = dir.listAll(); |
| - for (String file : files) { |
| - //System.out.println("file:" + file); |
| - try { |
| - IndexInput input = dir.openInput(file, newIOContext(random())); |
| - input.close(); |
| - } catch (FileNotFoundException | NoSuchFileException e) { |
| - // ignore |
| - } catch (IOException e) { |
| - if (e.getMessage().contains("still open for writing")) { |
| - // ignore |
| - } else { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - if (random().nextBoolean()) { |
| - break; |
| - } |
| - } |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - TheThread theThread = new TheThread("t1"); |
| - TheThread2 theThread2 = new TheThread2("t2"); |
| - theThread.start(); |
| - theThread2.start(); |
| - |
| - theThread.join(); |
| - theThread2.join(); |
| - |
| - dir.close(); |
| - } |
| - |
| - |
| - // Test that different instances of FSDirectory can coexist on the same |
| - // path, can read, write, and lock files. |
| - public void testDirectInstantiation() throws Exception { |
| - final File path = TestUtil.getTempDir("testDirectInstantiation"); |
| - |
| - final byte[] largeBuffer = new byte[random().nextInt(256*1024)], largeReadBuffer = new byte[largeBuffer.length]; |
| - for (int i = 0; i < largeBuffer.length; i++) { |
| - largeBuffer[i] = (byte) i; // automatically loops with modulo |
| - } |
| - |
| - final FSDirectory[] dirs = new FSDirectory[] { |
| - new SimpleFSDirectory(path, null), |
| - new NIOFSDirectory(path, null), |
| - new MMapDirectory(path, null) |
| - }; |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - String fname = "foo." + i; |
| - String lockname = "foo" + i + ".lck"; |
| - IndexOutput out = dir.createOutput(fname, newIOContext(random())); |
| - out.writeByte((byte)i); |
| - out.writeBytes(largeBuffer, largeBuffer.length); |
| - out.close(); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - d2.ensureOpen(); |
| - assertTrue(slowFileExists(d2, fname)); |
| - assertEquals(1 + largeBuffer.length, d2.fileLength(fname)); |
| - |
| - // don't do read tests if unmapping is not supported! |
| - if (d2 instanceof MMapDirectory && !((MMapDirectory) d2).getUseUnmap()) |
| - continue; |
| - |
| - IndexInput input = d2.openInput(fname, newIOContext(random())); |
| - assertEquals((byte)i, input.readByte()); |
| - // read array with buffering enabled |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, true); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - // read again without using buffer |
| - input.seek(1L); |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, false); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - input.close(); |
| - } |
| - |
| - // delete with a different dir |
| - dirs[(i+1)%dirs.length].deleteFile(fname); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - assertFalse(slowFileExists(d2, fname)); |
| - } |
| - |
| - Lock lock = dir.makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - Lock lock2 = d2.makeLock(lockname); |
| - try { |
| - assertFalse(lock2.obtain(1)); |
| - } catch (LockObtainFailedException e) { |
| - // OK |
| - } |
| - } |
| - |
| - lock.close(); |
| - |
| - // now lock with different dir |
| - lock = dirs[(i+1)%dirs.length].makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - lock.close(); |
| - } |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - dir.close(); |
| - assertFalse(dir.isOpen); |
| - } |
| - |
| - TestUtil.rmDir(path); |
| - } |
| - |
| - // LUCENE-1464 |
| - public void testDontCreate() throws Throwable { |
| - File path = new File(TEMP_DIR, "doesnotexist"); |
| - try { |
| - assertTrue(!path.exists()); |
| - Directory dir = new SimpleFSDirectory(path, null); |
| - assertTrue(!path.exists()); |
| - dir.close(); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testRAMDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(new RAMDirectory()); |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testFSDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(newFSDirectory(TestUtil.getTempDir("test"))); |
| - } |
| - |
| - // LUCENE-1468 |
| - private void checkDirectoryFilter(Directory dir) throws IOException { |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testCopySubdir() throws Throwable { |
| - File path = TestUtil.getTempDir("testsubdir"); |
| - try { |
| - path.mkdirs(); |
| - new File(path, "subdir").mkdirs(); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testNotDirectory() throws Throwable { |
| - File path = TestUtil.getTempDir("testnotdir"); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - try { |
| - IndexOutput out = fsDir.createOutput("afile", newIOContext(random())); |
| - out.close(); |
| - assertTrue(slowFileExists(fsDir, "afile")); |
| - try { |
| - new SimpleFSDirectory(new File(path, "afile"), null); |
| - fail("did not hit expected exception"); |
| - } catch (NoSuchDirectoryException nsde) { |
| - // Expected |
| - } |
| - } finally { |
| - fsDir.close(); |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| } |
| |
| Index: lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java (revision 0) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java (working copy) |
| @@ -0,0 +1,31 @@ |
| +package org.apache.lucene.store; |
| + |
| +/* |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| +import java.io.File; |
| + |
| +public class TestRateLimitedDirectoryWrapper extends BaseDirectoryTestCase { |
| + |
| + @Override |
| + protected Directory getDirectory(File path) { |
| + RateLimitedDirectoryWrapper dir = new RateLimitedDirectoryWrapper(newFSDirectory(path)); |
| + RateLimiter limiter = new RateLimiter.SimpleRateLimiter(.1 + 3*random().nextDouble()); |
| + dir.setRateLimiter(limiter, IOContext.Context.MERGE); |
| + return dir; |
| + } |
| +} |
| |
| Property changes on: lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| ## -0,0 +1 ## |
| +native |
| \ No newline at end of property |
| Index: lucene/core/src/test/org/apache/lucene/store/TestDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestDirectory.java (working copy) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestDirectory.java (working copy) |
| @@ -27,265 +27,15 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestDirectory extends LuceneTestCase { |
| - public void testDetectClose() throws Throwable { |
| - Directory[] dirs = new Directory[] { |
| - new RAMDirectory(), |
| - new SimpleFSDirectory(TEMP_DIR), |
| - new NIOFSDirectory(TEMP_DIR) |
| - }; |
| +public class TestDirectory extends BaseDirectoryTestCase { |
| |
| - for (Directory dir : dirs) { |
| - dir.close(); |
| - try { |
| - dir.createOutput("test", newIOContext(random())); |
| - fail("did not hit expected exception"); |
| - } catch (AlreadyClosedException ace) { |
| - } |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + if (random().nextBoolean()) { |
| + return newDirectory(); |
| + } else { |
| + return newFSDirectory(path); |
| } |
| } |
| - |
| - // test is occasionally very slow, i dont know why |
| - // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE |
| - @Nightly |
| - public void testThreadSafety() throws Exception { |
| - final BaseDirectoryWrapper dir = newDirectory(); |
| - dir.setCheckIndexOnClose(false); // we arent making an index |
| - if (dir instanceof MockDirectoryWrapper) { |
| - ((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow |
| - } |
| - |
| - if (VERBOSE) { |
| - System.out.println(dir); |
| - } |
| - |
| - class TheThread extends Thread { |
| - private String name; |
| - |
| - public TheThread(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 3000; i++) { |
| - String fileName = this.name + i; |
| - try { |
| - //System.out.println("create:" + fileName); |
| - IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| - output.close(); |
| - assertTrue(slowFileExists(dir, fileName)); |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - class TheThread2 extends Thread { |
| - private String name; |
| - |
| - public TheThread2(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 10000; i++) { |
| - try { |
| - String[] files = dir.listAll(); |
| - for (String file : files) { |
| - //System.out.println("file:" + file); |
| - try { |
| - IndexInput input = dir.openInput(file, newIOContext(random())); |
| - input.close(); |
| - } catch (FileNotFoundException | NoSuchFileException e) { |
| - // ignore |
| - } catch (IOException e) { |
| - if (e.getMessage().contains("still open for writing")) { |
| - // ignore |
| - } else { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - if (random().nextBoolean()) { |
| - break; |
| - } |
| - } |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - TheThread theThread = new TheThread("t1"); |
| - TheThread2 theThread2 = new TheThread2("t2"); |
| - theThread.start(); |
| - theThread2.start(); |
| - |
| - theThread.join(); |
| - theThread2.join(); |
| - |
| - dir.close(); |
| - } |
| - |
| - |
| - // Test that different instances of FSDirectory can coexist on the same |
| - // path, can read, write, and lock files. |
| - public void testDirectInstantiation() throws Exception { |
| - final File path = TestUtil.getTempDir("testDirectInstantiation"); |
| - |
| - final byte[] largeBuffer = new byte[random().nextInt(256*1024)], largeReadBuffer = new byte[largeBuffer.length]; |
| - for (int i = 0; i < largeBuffer.length; i++) { |
| - largeBuffer[i] = (byte) i; // automatically loops with modulo |
| - } |
| - |
| - final FSDirectory[] dirs = new FSDirectory[] { |
| - new SimpleFSDirectory(path, null), |
| - new NIOFSDirectory(path, null), |
| - new MMapDirectory(path, null) |
| - }; |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - String fname = "foo." + i; |
| - String lockname = "foo" + i + ".lck"; |
| - IndexOutput out = dir.createOutput(fname, newIOContext(random())); |
| - out.writeByte((byte)i); |
| - out.writeBytes(largeBuffer, largeBuffer.length); |
| - out.close(); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - d2.ensureOpen(); |
| - assertTrue(slowFileExists(d2, fname)); |
| - assertEquals(1 + largeBuffer.length, d2.fileLength(fname)); |
| - |
| - // don't do read tests if unmapping is not supported! |
| - if (d2 instanceof MMapDirectory && !((MMapDirectory) d2).getUseUnmap()) |
| - continue; |
| - |
| - IndexInput input = d2.openInput(fname, newIOContext(random())); |
| - assertEquals((byte)i, input.readByte()); |
| - // read array with buffering enabled |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, true); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - // read again without using buffer |
| - input.seek(1L); |
| - Arrays.fill(largeReadBuffer, (byte)0); |
| - input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, false); |
| - assertArrayEquals(largeBuffer, largeReadBuffer); |
| - input.close(); |
| - } |
| - |
| - // delete with a different dir |
| - dirs[(i+1)%dirs.length].deleteFile(fname); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - assertFalse(slowFileExists(d2, fname)); |
| - } |
| - |
| - Lock lock = dir.makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - |
| - for (int j=0; j<dirs.length; j++) { |
| - FSDirectory d2 = dirs[j]; |
| - Lock lock2 = d2.makeLock(lockname); |
| - try { |
| - assertFalse(lock2.obtain(1)); |
| - } catch (LockObtainFailedException e) { |
| - // OK |
| - } |
| - } |
| - |
| - lock.close(); |
| - |
| - // now lock with different dir |
| - lock = dirs[(i+1)%dirs.length].makeLock(lockname); |
| - assertTrue(lock.obtain()); |
| - lock.close(); |
| - } |
| - |
| - for (int i=0; i<dirs.length; i++) { |
| - FSDirectory dir = dirs[i]; |
| - dir.ensureOpen(); |
| - dir.close(); |
| - assertFalse(dir.isOpen); |
| - } |
| - |
| - TestUtil.rmDir(path); |
| - } |
| - |
| - // LUCENE-1464 |
| - public void testDontCreate() throws Throwable { |
| - File path = new File(TEMP_DIR, "doesnotexist"); |
| - try { |
| - assertTrue(!path.exists()); |
| - Directory dir = new SimpleFSDirectory(path, null); |
| - assertTrue(!path.exists()); |
| - dir.close(); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testRAMDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(new RAMDirectory()); |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testFSDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(newFSDirectory(TestUtil.getTempDir("test"))); |
| - } |
| - |
| - // LUCENE-1468 |
| - private void checkDirectoryFilter(Directory dir) throws IOException { |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testCopySubdir() throws Throwable { |
| - File path = TestUtil.getTempDir("testsubdir"); |
| - try { |
| - path.mkdirs(); |
| - new File(path, "subdir").mkdirs(); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testNotDirectory() throws Throwable { |
| - File path = TestUtil.getTempDir("testnotdir"); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - try { |
| - IndexOutput out = fsDir.createOutput("afile", newIOContext(random())); |
| - out.close(); |
| - assertTrue(slowFileExists(fsDir, "afile")); |
| - try { |
| - new SimpleFSDirectory(new File(path, "afile"), null); |
| - fail("did not hit expected exception"); |
| - } catch (NoSuchDirectoryException nsde) { |
| - // Expected |
| - } |
| - } finally { |
| - fsDir.close(); |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| } |
| |
| Index: lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java (revision 1583585) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java (working copy) |
| @@ -20,39 +20,40 @@ |
| import java.io.File; |
| import java.io.IOException; |
| |
| -import org.apache.lucene.document.Field; |
| -import org.apache.lucene.util.LuceneTestCase; |
| -import org.apache.lucene.util.TestUtil; |
| -import org.apache.lucene.util.TestUtil; |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.document.Document; |
| +import org.apache.lucene.document.Field; |
| import org.apache.lucene.index.DirectoryReader; |
| import org.apache.lucene.index.IndexReader; |
| import org.apache.lucene.index.IndexWriter; |
| +import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.index.IndexWriterConfig; |
| import org.apache.lucene.index.StoredDocument; |
| -import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.util.English; |
| +import org.apache.lucene.util.IOUtils; |
| +import org.apache.lucene.util.LuceneTestCase; |
| +import org.apache.lucene.util.TestUtil; |
| +import org.apache.lucene.util.TestUtil; |
| |
| /** |
| * JUnit testcase to test RAMDirectory. RAMDirectory itself is used in many testcases, |
| * but not one of them uses an different constructor other than the default constructor. |
| */ |
| -public class TestRAMDirectory extends LuceneTestCase { |
| +public class TestRAMDirectory extends BaseDirectoryTestCase { |
| |
| - private File indexDir = null; |
| + @Override |
| + protected Directory getDirectory(File path) { |
| + return new RAMDirectory(); |
| + } |
| |
| // add enough document so that the index will be larger than RAMDirectory.READ_BUFFER_SIZE |
| private final int docsToAdd = 500; |
| - |
| - // setup the index |
| - @Override |
| - public void setUp() throws Exception { |
| - super.setUp(); |
| - indexDir = TestUtil.getTempDir("RAMDirIndex"); |
| + |
| + private File buildIndex() throws IOException { |
| + File path = TestUtil.getTempDir("buildIndex"); |
| |
| - Directory dir = newFSDirectory(indexDir); |
| + Directory dir = newFSDirectory(path); |
| IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE)); |
| // add some documents |
| @@ -65,9 +66,27 @@ |
| assertEquals(docsToAdd, writer.maxDoc()); |
| writer.close(); |
| dir.close(); |
| + |
| + return path; |
| } |
| |
| + // LUCENE-1468 |
| + public void testCopySubdir() throws Throwable { |
| + File path = TestUtil.getTempDir("testsubdir"); |
| + Directory fsDir = null; |
| + try { |
| + path.mkdirs(); |
| + new File(path, "subdir").mkdirs(); |
| + fsDir = newFSDirectory(path); |
| + assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); |
| + } finally { |
| + TestUtil.rmDir(path); |
| + IOUtils.close(fsDir); |
| + } |
| + } |
| + |
| public void testRAMDirectory () throws IOException { |
| + File indexDir = buildIndex(); |
| |
| Directory dir = newFSDirectory(indexDir); |
| MockDirectoryWrapper ramDir = new MockDirectoryWrapper(random(), new RAMDirectory(dir, newIOContext(random()))); |
| @@ -99,6 +118,8 @@ |
| private final int docsPerThread = 40; |
| |
| public void testRAMDirectorySize() throws IOException, InterruptedException { |
| + |
| + File indexDir = buildIndex(); |
| |
| Directory dir = newFSDirectory(indexDir); |
| final MockDirectoryWrapper ramDir = new MockDirectoryWrapper(random(), new RAMDirectory(dir, newIOContext(random()))); |
| @@ -138,52 +159,4 @@ |
| |
| writer.close(); |
| } |
| - |
| - @Override |
| - public void tearDown() throws Exception { |
| - // cleanup |
| - if (indexDir != null && indexDir.exists()) { |
| - rmDir (indexDir); |
| - } |
| - super.tearDown(); |
| - } |
| - |
| - // LUCENE-1196 |
| - public void testIllegalEOF() throws Exception { |
| - RAMDirectory dir = new RAMDirectory(); |
| - IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| - byte[] b = new byte[1024]; |
| - o.writeBytes(b, 0, 1024); |
| - o.close(); |
| - IndexInput i = dir.openInput("out", newIOContext(random())); |
| - i.seek(1024); |
| - i.close(); |
| - dir.close(); |
| - } |
| - |
| - private void rmDir(File dir) { |
| - File[] files = dir.listFiles(); |
| - for (int i = 0; i < files.length; i++) { |
| - files[i].delete(); |
| - } |
| - dir.delete(); |
| - } |
| - |
| - // LUCENE-2852 |
| - public void testSeekToEOFThenBack() throws Exception { |
| - RAMDirectory dir = new RAMDirectory(); |
| - |
| - IndexOutput o = dir.createOutput("out", newIOContext(random())); |
| - byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE]; |
| - o.writeBytes(bytes, 0, bytes.length); |
| - o.close(); |
| - |
| - IndexInput i = dir.openInput("out", newIOContext(random())); |
| - i.seek(2*RAMInputStream.BUFFER_SIZE-1); |
| - i.seek(3*RAMInputStream.BUFFER_SIZE); |
| - i.seek(RAMInputStream.BUFFER_SIZE); |
| - i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE); |
| - i.close(); |
| - dir.close(); |
| - } |
| } |
| Index: lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (revision 1583585) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (working copy) |
| @@ -39,8 +39,15 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestNRTCachingDirectory extends LuceneTestCase { |
| +public class TestNRTCachingDirectory extends BaseDirectoryTestCase { |
| |
| + @Override |
| + protected Directory getDirectory(File path) { |
| + return new NRTCachingDirectory(newFSDirectory(path), |
| + .1 + 2.0*random().nextDouble(), |
| + .1 + 5.0*random().nextDouble()); |
| + } |
| + |
| public void testNRTAndCommit() throws Exception { |
| Directory dir = newDirectory(); |
| NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0); |
| @@ -112,72 +119,4 @@ |
| IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer); |
| IndexWriter writer = new IndexWriter(cachedFSDir, conf); |
| } |
| - |
| - public void testDeleteFile() throws Exception { |
| - Directory dir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0); |
| - dir.createOutput("foo.txt", IOContext.DEFAULT).close(); |
| - dir.deleteFile("foo.txt"); |
| - assertEquals(0, dir.listAll().length); |
| - dir.close(); |
| - } |
| - |
| - // LUCENE-3382 -- make sure we get exception if the directory really does not exist. |
| - public void testNoDir() throws Throwable { |
| - File tempDir = TestUtil.getTempDir("doesnotexist"); |
| - TestUtil.rmDir(tempDir); |
| - Directory dir = new NRTCachingDirectory(newFSDirectory(tempDir), 2.0, 25.0); |
| - try { |
| - DirectoryReader.open(dir); |
| - fail("did not hit expected exception"); |
| - } catch (NoSuchDirectoryException nsde) { |
| - // expected |
| - } |
| - dir.close(); |
| - } |
| - |
| - // LUCENE-3382 test that we can add a file, and then when we call list() we get it back |
| - public void testDirectoryFilter() throws IOException { |
| - Directory dir = new NRTCachingDirectory(newFSDirectory(TestUtil.getTempDir("foo")), 2.0, 25.0); |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-3382 test that delegate compound files correctly. |
| - public void testCompoundFileAppendTwice() throws IOException { |
| - Directory newDir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0); |
| - CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true); |
| - createSequenceFile(newDir, "d1", (byte) 0, 15); |
| - IndexOutput out = csw.createOutput("d.xyz", newIOContext(random())); |
| - out.writeInt(0); |
| - out.close(); |
| - assertEquals(1, csw.listAll().length); |
| - assertEquals("d.xyz", csw.listAll()[0]); |
| - |
| - csw.close(); |
| - |
| - CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false); |
| - assertEquals(1, cfr.listAll().length); |
| - assertEquals("d.xyz", cfr.listAll()[0]); |
| - cfr.close(); |
| - newDir.close(); |
| - } |
| - |
| - /** Creates a file of the specified size with sequential data. The first |
| - * byte is written as the start byte provided. All subsequent bytes are |
| - * computed as start + offset where offset is the number of the byte. |
| - */ |
| - private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException { |
| - IndexOutput os = dir.createOutput(name, newIOContext(random())); |
| - for (int i=0; i < size; i++) { |
| - os.writeByte(start); |
| - start ++; |
| - } |
| - os.close(); |
| - } |
| } |
| Index: lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java (revision 1583585) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java (working copy) |
| @@ -18,6 +18,7 @@ |
| */ |
| |
| import java.io.File; |
| +import java.io.IOException; |
| import java.util.Random; |
| |
| import org.apache.lucene.analysis.MockAnalyzer; |
| @@ -37,8 +38,13 @@ |
| * values, it's necessary to access a file > |
| * Integer.MAX_VALUE in size using multiple byte buffers. |
| */ |
| -public class TestMultiMMap extends LuceneTestCase { |
| +public class TestMultiMMap extends BaseDirectoryTestCase { |
| File workDir; |
| + |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + return new MMapDirectory(path, null, 1<<TestUtil.nextInt(random(), 10, 28)); |
| + } |
| |
| @Override |
| public void setUp() throws Exception { |
| Index: lucene/core/src/test/org/apache/lucene/store/TestSimpleFSDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestSimpleFSDirectory.java (working copy) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestSimpleFSDirectory.java (working copy) |
| @@ -27,110 +27,13 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestDirectory extends LuceneTestCase { |
| - public void testDetectClose() throws Throwable { |
| - Directory[] dirs = new Directory[] { |
| - new RAMDirectory(), |
| - new SimpleFSDirectory(TEMP_DIR), |
| - new NIOFSDirectory(TEMP_DIR) |
| - }; |
| +public class TestSimpleFSDirectory extends BaseDirectoryTestCase { |
| |
| - for (Directory dir : dirs) { |
| - dir.close(); |
| - try { |
| - dir.createOutput("test", newIOContext(random())); |
| - fail("did not hit expected exception"); |
| - } catch (AlreadyClosedException ace) { |
| - } |
| - } |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + return new SimpleFSDirectory(path); |
| } |
| - |
| - // test is occasionally very slow, i dont know why |
| - // try this seed: 7D7E036AD12927F5:93333EF9E6DE44DE |
| - @Nightly |
| - public void testThreadSafety() throws Exception { |
| - final BaseDirectoryWrapper dir = newDirectory(); |
| - dir.setCheckIndexOnClose(false); // we arent making an index |
| - if (dir instanceof MockDirectoryWrapper) { |
| - ((MockDirectoryWrapper)dir).setThrottling(Throttling.NEVER); // makes this test really slow |
| - } |
| - |
| - if (VERBOSE) { |
| - System.out.println(dir); |
| - } |
| |
| - class TheThread extends Thread { |
| - private String name; |
| - |
| - public TheThread(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 3000; i++) { |
| - String fileName = this.name + i; |
| - try { |
| - //System.out.println("create:" + fileName); |
| - IndexOutput output = dir.createOutput(fileName, newIOContext(random())); |
| - output.close(); |
| - assertTrue(slowFileExists(dir, fileName)); |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - class TheThread2 extends Thread { |
| - private String name; |
| - |
| - public TheThread2(String name) { |
| - this.name = name; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - for (int i = 0; i < 10000; i++) { |
| - try { |
| - String[] files = dir.listAll(); |
| - for (String file : files) { |
| - //System.out.println("file:" + file); |
| - try { |
| - IndexInput input = dir.openInput(file, newIOContext(random())); |
| - input.close(); |
| - } catch (FileNotFoundException | NoSuchFileException e) { |
| - // ignore |
| - } catch (IOException e) { |
| - if (e.getMessage().contains("still open for writing")) { |
| - // ignore |
| - } else { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - if (random().nextBoolean()) { |
| - break; |
| - } |
| - } |
| - } catch (IOException e) { |
| - throw new RuntimeException(e); |
| - } |
| - } |
| - } |
| - }; |
| - |
| - TheThread theThread = new TheThread("t1"); |
| - TheThread2 theThread2 = new TheThread2("t2"); |
| - theThread.start(); |
| - theThread2.start(); |
| - |
| - theThread.join(); |
| - theThread2.join(); |
| - |
| - dir.close(); |
| - } |
| - |
| - |
| // Test that different instances of FSDirectory can coexist on the same |
| // path, can read, write, and lock files. |
| public void testDirectInstantiation() throws Exception { |
| @@ -220,55 +123,7 @@ |
| TestUtil.rmDir(path); |
| } |
| |
| - // LUCENE-1464 |
| - public void testDontCreate() throws Throwable { |
| - File path = new File(TEMP_DIR, "doesnotexist"); |
| - try { |
| - assertTrue(!path.exists()); |
| - Directory dir = new SimpleFSDirectory(path, null); |
| - assertTrue(!path.exists()); |
| - dir.close(); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| // LUCENE-1468 |
| - public void testRAMDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(new RAMDirectory()); |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testFSDirectoryFilter() throws IOException { |
| - checkDirectoryFilter(newFSDirectory(TestUtil.getTempDir("test"))); |
| - } |
| - |
| - // LUCENE-1468 |
| - private void checkDirectoryFilter(Directory dir) throws IOException { |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| - public void testCopySubdir() throws Throwable { |
| - File path = TestUtil.getTempDir("testsubdir"); |
| - try { |
| - path.mkdirs(); |
| - new File(path, "subdir").mkdirs(); |
| - Directory fsDir = new SimpleFSDirectory(path, null); |
| - assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); |
| - } finally { |
| - TestUtil.rmDir(path); |
| - } |
| - } |
| - |
| - // LUCENE-1468 |
| public void testNotDirectory() throws Throwable { |
| File path = TestUtil.getTempDir("testnotdir"); |
| Directory fsDir = new SimpleFSDirectory(path, null); |
| Index: lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java (revision 1583585) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java (working copy) |
| @@ -1,173 +0,0 @@ |
| -package org.apache.lucene.store; |
| - |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one or more |
| - * contributor license agreements. See the NOTICE file distributed with |
| - * this work for additional information regarding copyright ownership. |
| - * The ASF licenses this file to You under the Apache License, Version 2.0 |
| - * (the "License"); you may not use this file except in compliance with |
| - * the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, software |
| - * distributed under the License is distributed on an "AS IS" BASIS, |
| - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| - * See the License for the specific language governing permissions and |
| - * limitations under the License. |
| - */ |
| - |
| -import java.io.IOException; |
| - |
| -import org.apache.lucene.util.LuceneTestCase; |
| -import org.apache.lucene.util.TestUtil; |
| - |
| -import org.junit.Test; |
| - |
| -public class TestCopyBytes extends LuceneTestCase { |
| - |
| - private byte value(int idx) { |
| - return (byte) ((idx % 256) * (1 + (idx / 256))); |
| - } |
| - |
| - @Test |
| - public void testCopyBytes() throws Exception { |
| - int num = atLeast(10); |
| - for (int iter = 0; iter < num; iter++) { |
| - Directory dir = newDirectory(); |
| - if (VERBOSE) { |
| - System.out.println("TEST: iter=" + iter + " dir=" + dir); |
| - } |
| - |
| - // make random file |
| - IndexOutput out = dir.createOutput("test", newIOContext(random())); |
| - byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)]; |
| - final int size = TestUtil.nextInt(random(), 1, 1777777); |
| - int upto = 0; |
| - int byteUpto = 0; |
| - while (upto < size) { |
| - bytes[byteUpto++] = value(upto); |
| - upto++; |
| - if (byteUpto == bytes.length) { |
| - out.writeBytes(bytes, 0, bytes.length); |
| - byteUpto = 0; |
| - } |
| - } |
| - |
| - out.writeBytes(bytes, 0, byteUpto); |
| - assertEquals(size, out.getFilePointer()); |
| - out.close(); |
| - assertEquals(size, dir.fileLength("test")); |
| - |
| - // copy from test -> test2 |
| - final IndexInput in = dir.openInput("test", newIOContext(random())); |
| - |
| - out = dir.createOutput("test2", newIOContext(random())); |
| - |
| - upto = 0; |
| - while (upto < size) { |
| - if (random().nextBoolean()) { |
| - out.writeByte(in.readByte()); |
| - upto++; |
| - } else { |
| - final int chunk = Math.min( |
| - TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| - out.copyBytes(in, chunk); |
| - upto += chunk; |
| - } |
| - } |
| - assertEquals(size, upto); |
| - out.close(); |
| - in.close(); |
| - |
| - // verify |
| - IndexInput in2 = dir.openInput("test2", newIOContext(random())); |
| - upto = 0; |
| - while (upto < size) { |
| - if (random().nextBoolean()) { |
| - final byte v = in2.readByte(); |
| - assertEquals(value(upto), v); |
| - upto++; |
| - } else { |
| - final int limit = Math.min( |
| - TestUtil.nextInt(random(), 1, bytes.length), size - upto); |
| - in2.readBytes(bytes, 0, limit); |
| - for (int byteIdx = 0; byteIdx < limit; byteIdx++) { |
| - assertEquals(value(upto), bytes[byteIdx]); |
| - upto++; |
| - } |
| - } |
| - } |
| - in2.close(); |
| - |
| - dir.deleteFile("test"); |
| - dir.deleteFile("test2"); |
| - |
| - dir.close(); |
| - } |
| - } |
| - |
| - // LUCENE-3541 |
| - public void testCopyBytesWithThreads() throws Exception { |
| - int datalen = TestUtil.nextInt(random(), 101, 10000); |
| - byte data[] = new byte[datalen]; |
| - random().nextBytes(data); |
| - |
| - Directory d = newDirectory(); |
| - IndexOutput output = d.createOutput("data", IOContext.DEFAULT); |
| - output.writeBytes(data, 0, datalen); |
| - output.close(); |
| - |
| - IndexInput input = d.openInput("data", IOContext.DEFAULT); |
| - IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT); |
| - // copy our 100-byte header |
| - outputHeader.copyBytes(input, 100); |
| - outputHeader.close(); |
| - |
| - // now make N copies of the remaining bytes |
| - CopyThread copies[] = new CopyThread[10]; |
| - for (int i = 0; i < copies.length; i++) { |
| - copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT)); |
| - } |
| - |
| - for (int i = 0; i < copies.length; i++) { |
| - copies[i].start(); |
| - } |
| - |
| - for (int i = 0; i < copies.length; i++) { |
| - copies[i].join(); |
| - } |
| - |
| - for (int i = 0; i < copies.length; i++) { |
| - IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT); |
| - byte[] dataCopy = new byte[datalen]; |
| - System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing |
| - copiedData.readBytes(dataCopy, 100, datalen-100); |
| - assertArrayEquals(data, dataCopy); |
| - copiedData.close(); |
| - } |
| - input.close(); |
| - d.close(); |
| - |
| - } |
| - |
| - static class CopyThread extends Thread { |
| - final IndexInput src; |
| - final IndexOutput dst; |
| - |
| - CopyThread(IndexInput src, IndexOutput dst) { |
| - this.src = src; |
| - this.dst = dst; |
| - } |
| - |
| - @Override |
| - public void run() { |
| - try { |
| - dst.copyBytes(src, src.length()-100); |
| - dst.close(); |
| - } catch (IOException ex) { |
| - throw new RuntimeException(ex); |
| - } |
| - } |
| - } |
| -} |
| Index: lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (revision 1583585) |
| +++ lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (working copy) |
| @@ -35,7 +35,8 @@ |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| |
| -public class TestFileSwitchDirectory extends LuceneTestCase { |
| +public class TestFileSwitchDirectory extends BaseDirectoryTestCase { |
| + |
| /** |
| * Test if writing doc stores to disk and everything else to ram works. |
| */ |
| @@ -115,51 +116,28 @@ |
| } |
| dir.close(); |
| } |
| - |
| - // LUCENE-3380 test that we can add a file, and then when we call list() we get it back |
| - public void testDirectoryFilter() throws IOException { |
| - Directory dir = newFSSwitchDirectory(Collections.<String>emptySet()); |
| - String name = "file"; |
| - try { |
| - dir.createOutput(name, newIOContext(random())).close(); |
| - assertTrue(slowFileExists(dir, name)); |
| - assertTrue(Arrays.asList(dir.listAll()).contains(name)); |
| - } finally { |
| - dir.close(); |
| + |
| + @Override |
| + protected Directory getDirectory(File path) throws IOException { |
| + Set<String> extensions = new HashSet<String>(); |
| + if (random().nextBoolean()) { |
| + extensions.add("cfs"); |
| } |
| + if (random().nextBoolean()) { |
| + extensions.add("prx"); |
| + } |
| + if (random().nextBoolean()) { |
| + extensions.add("frq"); |
| + } |
| + if (random().nextBoolean()) { |
| + extensions.add("tip"); |
| + } |
| + if (random().nextBoolean()) { |
| + extensions.add("tim"); |
| + } |
| + if (random().nextBoolean()) { |
| + extensions.add("del"); |
| + } |
| + return newFSSwitchDirectory(extensions); |
| } |
| - |
| - // LUCENE-3380 test that delegate compound files correctly. |
| - public void testCompoundFileAppendTwice() throws IOException { |
| - Directory newDir = newFSSwitchDirectory(Collections.singleton("cfs")); |
| - CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true); |
| - createSequenceFile(newDir, "d1", (byte) 0, 15); |
| - IndexOutput out = csw.createOutput("d.xyz", newIOContext(random())); |
| - out.writeInt(0); |
| - out.close(); |
| - assertEquals(1, csw.listAll().length); |
| - assertEquals("d.xyz", csw.listAll()[0]); |
| - |
| - csw.close(); |
| - |
| - CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false); |
| - assertEquals(1, cfr.listAll().length); |
| - assertEquals("d.xyz", cfr.listAll()[0]); |
| - cfr.close(); |
| - newDir.close(); |
| - } |
| - |
| - /** Creates a file of the specified size with sequential data. The first |
| - * byte is written as the start byte provided. All subsequent bytes are |
| - * computed as start + offset where offset is the number of the byte. |
| - */ |
| - private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException { |
| - IndexOutput os = dir.createOutput(name, newIOContext(random())); |
| - for (int i=0; i < size; i++) { |
| - os.writeByte(start); |
| - start ++; |
| - } |
| - os.close(); |
| - } |
| - |
| } |
| Index: lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java (revision 1583585) |
| +++ lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java (working copy) |
| @@ -276,4 +276,9 @@ |
| |
| return buffers; |
| } |
| + |
| + @Override |
| + public String toString() { |
| + return super.toString() + " chunkSize=" + getMaxChunkSize(); |
| + } |
| } |
| Index: lucene/core/src/java/org/apache/lucene/store/IndexOutput.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (revision 1583585) |
| +++ lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (working copy) |
| @@ -31,6 +31,7 @@ |
| */ |
| public abstract class IndexOutput extends DataOutput implements Closeable { |
| |
| + // TODO: can we remove this? |
| /** Forces any buffered output to be written. */ |
| public abstract void flush() throws IOException; |
| |
| @@ -46,9 +47,12 @@ |
| /** Returns the current checksum of bytes written so far */ |
| public abstract long getChecksum() throws IOException; |
| |
| + // TODO: can we remove this? It's always the same as |
| + // .getFilePointer? |
| /** The number of bytes in the file. */ |
| public abstract long length() throws IOException; |
| |
| + // TODO: remove this |
| /** Set the file length. By default, this method does |
| * nothing (it's optional for a Directory to implement |
| * it). But, certain Directory implementations (for |
| Index: lucene/common-build.xml |
| =================================================================== |
| --- lucene/common-build.xml (revision 1583585) |
| +++ lucene/common-build.xml (working copy) |
| @@ -915,6 +915,10 @@ |
| <enable package="org.apache.solr"/> |
| </assertions> |
| |
| + <!-- TODO: generalize this for non-unix --> |
| + <!-- Add any native extensions to LD_LIBRARY_PATH: --> |
| + <env key="LD_LIBRARY_PATH" value="/l/fixnativeunix/lucene/build/native/"></env> |
| + |
| <!-- JVM arguments and system properties. --> |
| <jvmarg line="${args}"/> |
| <jvmarg line="${tests.heapdump.args}"/> |
