lucene/src/test/org/apache/lucene/index/TestCompoundFile.java - manifoldcf-integration-solr-3.x - Git at Google

 package org.apache.lucene.index;

 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 import java.io.IOException;
 import java.io.File;

 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.MockDirectoryWrapper.Failure;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.store._TestHelper;
 import org.apache.lucene.util._TestUtil;


 public class TestCompoundFile extends LuceneTestCase
 {
     private Directory dir;

     @Override
     public void setUp() throws Exception {
        super.setUp();
        File file = _TestUtil.getTempDir("testIndex");
        // use a simple FSDir here, to be sure to have SimpleFSInputs
        dir = new SimpleFSDirectory(file,null);
     }

     @Override
     public void tearDown() throws Exception {
        dir.close();
        super.tearDown();
     }

     /** Creates a file of the specified size with random data. */
     private void createRandomFile(Directory dir, String name, int size)
     throws IOException
     {
         IndexOutput os = dir.createOutput(name);
         for (int i=0; i<size; i++) {
             byte b = (byte) (Math.random() * 256);
             os.writeByte(b);
         }
         os.close();
     }

     /** Creates a file of the specified size with sequential data. The first
      *  byte is written as the start byte provided. All subsequent bytes are
      *  computed as start + offset where offset is the number of the byte.
      */
     private void createSequenceFile(Directory dir,
                                     String name,
                                     byte start,
                                     int size)
     throws IOException
     {
         IndexOutput os = dir.createOutput(name);
         for (int i=0; i < size; i++) {
             os.writeByte(start);
             start ++;
         }
         os.close();
     }


     private void assertSameStreams(String msg,
                                    IndexInput expected,
                                    IndexInput test)
     throws IOException
     {
         assertNotNull(msg + " null expected", expected);
         assertNotNull(msg + " null test", test);
         assertEquals(msg + " length", expected.length(), test.length());
         assertEquals(msg + " position", expected.getFilePointer(),
                                         test.getFilePointer());

         byte expectedBuffer[] = new byte[512];
         byte testBuffer[] = new byte[expectedBuffer.length];

         long remainder = expected.length() - expected.getFilePointer();
         while(remainder > 0) {
             int readLen = (int) Math.min(remainder, expectedBuffer.length);
             expected.readBytes(expectedBuffer, 0, readLen);
             test.readBytes(testBuffer, 0, readLen);
             assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
                 testBuffer, 0, readLen);
             remainder -= readLen;
         }
     }


     private void assertSameStreams(String msg,
                                    IndexInput expected,
                                    IndexInput actual,
                                    long seekTo)
     throws IOException
     {
         if(seekTo >= 0 && seekTo < expected.length())
         {
             expected.seek(seekTo);
             actual.seek(seekTo);
             assertSameStreams(msg + ", seek(mid)", expected, actual);
         }
     }


     private void assertSameSeekBehavior(String msg,
                                         IndexInput expected,
                                         IndexInput actual)
     throws IOException
     {
         // seek to 0
         long point = 0;
         assertSameStreams(msg + ", seek(0)", expected, actual, point);

         // seek to middle
         point = expected.length() / 2l;
         assertSameStreams(msg + ", seek(mid)", expected, actual, point);

         // seek to end - 2
         point = expected.length() - 2;
         assertSameStreams(msg + ", seek(end-2)", expected, actual, point);

         // seek to end - 1
         point = expected.length() - 1;
         assertSameStreams(msg + ", seek(end-1)", expected, actual, point);

         // seek to the end
         point = expected.length();
         assertSameStreams(msg + ", seek(end)", expected, actual, point);

         // seek past end
         point = expected.length() + 1;
         assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
     }


     private void assertEqualArrays(String msg,
                                    byte[] expected,
                                    byte[] test,
                                    int start,
                                    int len)
     {
         assertNotNull(msg + " null expected", expected);
         assertNotNull(msg + " null test", test);

         for (int i=start; i<len; i++) {
             assertEquals(msg + " " + i, expected[i], test[i]);
         }
     }


     // ===========================================================
     //  Tests of the basic CompoundFile functionality
     // ===========================================================


     /** This test creates compound file based on a single file.
      *  Files of different sizes are tested: 0, 1, 10, 100 bytes.
      */
     public void testSingleFile() throws IOException {
         int data[] = new int[] { 0, 1, 10, 100 };
         for (int i=0; i<data.length; i++) {
             String name = "t" + data[i];
             createSequenceFile(dir, name, (byte) 0, data[i]);
             CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
             csw.addFile(name);
             csw.close();

             CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
             IndexInput expected = dir.openInput(name);
             IndexInput actual = csr.openInput(name);
             assertSameStreams(name, expected, actual);
             assertSameSeekBehavior(name, expected, actual);
             expected.close();
             actual.close();
             csr.close();
         }
     }


     /** This test creates compound file based on two files.
      *
      */
     public void testTwoFiles() throws IOException {
         createSequenceFile(dir, "d1", (byte) 0, 15);
         createSequenceFile(dir, "d2", (byte) 0, 114);

         CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
         csw.addFile("d1");
         csw.addFile("d2");
         csw.close();

         CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
         IndexInput expected = dir.openInput("d1");
         IndexInput actual = csr.openInput("d1");
         assertSameStreams("d1", expected, actual);
         assertSameSeekBehavior("d1", expected, actual);
         expected.close();
         actual.close();

         expected = dir.openInput("d2");
         actual = csr.openInput("d2");
         assertSameStreams("d2", expected, actual);
         assertSameSeekBehavior("d2", expected, actual);
         expected.close();
         actual.close();
         csr.close();
     }

     /** This test creates a compound file based on a large number of files of
      *  various length. The file content is generated randomly. The sizes range
      *  from 0 to 1Mb. Some of the sizes are selected to test the buffering
      *  logic in the file reading code. For this the chunk variable is set to
      *  the length of the buffer used internally by the compound file logic.
      */
     public void testRandomFiles() throws IOException {
         // Setup the test segment
         String segment = "test";
         int chunk = 1024; // internal buffer size used by the stream
         createRandomFile(dir, segment + ".zero", 0);
         createRandomFile(dir, segment + ".one", 1);
         createRandomFile(dir, segment + ".ten", 10);
         createRandomFile(dir, segment + ".hundred", 100);
         createRandomFile(dir, segment + ".big1", chunk);
         createRandomFile(dir, segment + ".big2", chunk - 1);
         createRandomFile(dir, segment + ".big3", chunk + 1);
         createRandomFile(dir, segment + ".big4", 3 * chunk);
         createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
         createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
         createRandomFile(dir, segment + ".big7", 1000 * chunk);

         // Setup extraneous files
         createRandomFile(dir, "onetwothree", 100);
         createRandomFile(dir, segment + ".notIn", 50);
         createRandomFile(dir, segment + ".notIn2", 51);

         // Now test
         CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
         final String data[] = new String[] {
             ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
             ".big4", ".big5", ".big6", ".big7"
         };
         for (int i=0; i<data.length; i++) {
             csw.addFile(segment + data[i]);
         }
         csw.close();

         CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
         for (int i=0; i<data.length; i++) {
             IndexInput check = dir.openInput(segment + data[i]);
             IndexInput test = csr.openInput(segment + data[i]);
             assertSameStreams(data[i], check, test);
             assertSameSeekBehavior(data[i], check, test);
             test.close();
             check.close();
         }
         csr.close();
     }


     /** Setup a larger compound file with a number of components, each of
      *  which is a sequential file (so that we can easily tell that we are
      *  reading in the right byte). The methods sets up 20 files - f0 to f19,
      *  the size of each file is 1000 bytes.
      */
     private void setUp_2() throws IOException {
         CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
         for (int i=0; i<20; i++) {
             createSequenceFile(dir, "f" + i, (byte) 0, 2000);
             cw.addFile("f" + i);
         }
         cw.close();
     }


     public void testReadAfterClose() throws IOException {
         demo_FSIndexInputBug(dir, "test");
     }

     private void demo_FSIndexInputBug(Directory fsdir, String file)
     throws IOException
     {
         // Setup the test file - we need more than 1024 bytes
         IndexOutput os = fsdir.createOutput(file);
         for(int i=0; i<2000; i++) {
             os.writeByte((byte) i);
         }
         os.close();

         IndexInput in = fsdir.openInput(file);

         // This read primes the buffer in IndexInput
         in.readByte();

         // Close the file
         in.close();

         // ERROR: this call should fail, but succeeds because the buffer
         // is still filled
         in.readByte();

         // ERROR: this call should fail, but succeeds for some reason as well
         in.seek(1099);

         try {
             // OK: this call correctly fails. We are now past the 1024 internal
             // buffer, so an actual IO is attempted, which fails
             in.readByte();
             fail("expected readByte() to throw exception");
         } catch (IOException e) {
           // expected exception
         }
     }


     static boolean isCSIndexInput(IndexInput is) {
         return is instanceof CompoundFileReader.CSIndexInput;
     }

     static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
         if (isCSIndexInput(is)) {
             CompoundFileReader.CSIndexInput cis =
             (CompoundFileReader.CSIndexInput) is;

             return _TestHelper.isSimpleFSIndexInputOpen(cis.base);
         } else {
             return false;
         }
     }


     public void testClonedStreamsClosing() throws IOException {
         setUp_2();
         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

         // basic clone
         IndexInput expected = dir.openInput("f11");

         // this test only works for FSIndexInput
         assertTrue(_TestHelper.isSimpleFSIndexInput(expected));
         assertTrue(_TestHelper.isSimpleFSIndexInputOpen(expected));

         IndexInput one = cr.openInput("f11");
         assertTrue(isCSIndexInputOpen(one));

         IndexInput two = (IndexInput) one.clone();
         assertTrue(isCSIndexInputOpen(two));

         assertSameStreams("basic clone one", expected, one);
         expected.seek(0);
         assertSameStreams("basic clone two", expected, two);

         // Now close the first stream
         one.close();
         assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));

         // The following should really fail since we couldn't expect to
         // access a file once close has been called on it (regardless of
         // buffering and/or clone magic)
         expected.seek(0);
         two.seek(0);
         assertSameStreams("basic clone two/2", expected, two);


         // Now close the compound reader
         cr.close();
         assertFalse("Now closed one", isCSIndexInputOpen(one));
         assertFalse("Now closed two", isCSIndexInputOpen(two));

         // The following may also fail since the compound stream is closed
         expected.seek(0);
         two.seek(0);
         //assertSameStreams("basic clone two/3", expected, two);


         // Now close the second clone
         two.close();
         expected.seek(0);
         two.seek(0);
         //assertSameStreams("basic clone two/4", expected, two);

         expected.close();
     }


     /** This test opens two files from a compound stream and verifies that
      *  their file positions are independent of each other.
      */
     public void testRandomAccess() throws IOException {
         setUp_2();
         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

         // Open two files
         IndexInput e1 = dir.openInput("f11");
         IndexInput e2 = dir.openInput("f3");

         IndexInput a1 = cr.openInput("f11");
         IndexInput a2 = dir.openInput("f3");

         // Seek the first pair
         e1.seek(100);
         a1.seek(100);
         assertEquals(100, e1.getFilePointer());
         assertEquals(100, a1.getFilePointer());
         byte be1 = e1.readByte();
         byte ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now seek the second pair
         e2.seek(1027);
         a2.seek(1027);
         assertEquals(1027, e2.getFilePointer());
         assertEquals(1027, a2.getFilePointer());
         byte be2 = e2.readByte();
         byte ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Now make sure the first one didn't move
         assertEquals(101, e1.getFilePointer());
         assertEquals(101, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now more the first one again, past the buffer length
         e1.seek(1910);
         a1.seek(1910);
         assertEquals(1910, e1.getFilePointer());
         assertEquals(1910, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now make sure the second set didn't move
         assertEquals(1028, e2.getFilePointer());
         assertEquals(1028, a2.getFilePointer());
         be2 = e2.readByte();
         ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Move the second set back, again cross the buffer size
         e2.seek(17);
         a2.seek(17);
         assertEquals(17, e2.getFilePointer());
         assertEquals(17, a2.getFilePointer());
         be2 = e2.readByte();
         ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Finally, make sure the first set didn't move
         // Now make sure the first one didn't move
         assertEquals(1911, e1.getFilePointer());
         assertEquals(1911, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         e1.close();
         e2.close();
         a1.close();
         a2.close();
         cr.close();
     }

     /** This test opens two files from a compound stream and verifies that
      *  their file positions are independent of each other.
      */
     public void testRandomAccessClones() throws IOException {
         setUp_2();
         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

         // Open two files
         IndexInput e1 = cr.openInput("f11");
         IndexInput e2 = cr.openInput("f3");

         IndexInput a1 = (IndexInput) e1.clone();
         IndexInput a2 = (IndexInput) e2.clone();

         // Seek the first pair
         e1.seek(100);
         a1.seek(100);
         assertEquals(100, e1.getFilePointer());
         assertEquals(100, a1.getFilePointer());
         byte be1 = e1.readByte();
         byte ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now seek the second pair
         e2.seek(1027);
         a2.seek(1027);
         assertEquals(1027, e2.getFilePointer());
         assertEquals(1027, a2.getFilePointer());
         byte be2 = e2.readByte();
         byte ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Now make sure the first one didn't move
         assertEquals(101, e1.getFilePointer());
         assertEquals(101, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now more the first one again, past the buffer length
         e1.seek(1910);
         a1.seek(1910);
         assertEquals(1910, e1.getFilePointer());
         assertEquals(1910, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         // Now make sure the second set didn't move
         assertEquals(1028, e2.getFilePointer());
         assertEquals(1028, a2.getFilePointer());
         be2 = e2.readByte();
         ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Move the second set back, again cross the buffer size
         e2.seek(17);
         a2.seek(17);
         assertEquals(17, e2.getFilePointer());
         assertEquals(17, a2.getFilePointer());
         be2 = e2.readByte();
         ba2 = a2.readByte();
         assertEquals(be2, ba2);

         // Finally, make sure the first set didn't move
         // Now make sure the first one didn't move
         assertEquals(1911, e1.getFilePointer());
         assertEquals(1911, a1.getFilePointer());
         be1 = e1.readByte();
         ba1 = a1.readByte();
         assertEquals(be1, ba1);

         e1.close();
         e2.close();
         a1.close();
         a2.close();
         cr.close();
     }


     public void testFileNotFound() throws IOException {
         setUp_2();
         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

         // Open two files
         try {
             cr.openInput("bogus");
             fail("File not found");

         } catch (IOException e) {
             /* success */
             //System.out.println("SUCCESS: File Not Found: " + e);
         }

         cr.close();
     }


     public void testReadPastEOF() throws IOException {
         setUp_2();
         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
         IndexInput is = cr.openInput("f2");
         is.seek(is.length() - 10);
         byte b[] = new byte[100];
         is.readBytes(b, 0, 10);

         try {
             is.readByte();
             fail("Single byte read past end of file");
         } catch (IOException e) {
             /* success */
             //System.out.println("SUCCESS: single byte read past end of file: " + e);
         }

         is.seek(is.length() - 10);
         try {
             is.readBytes(b, 0, 50);
             fail("Block read past end of file");
         } catch (IOException e) {
             /* success */
             //System.out.println("SUCCESS: block read past end of file: " + e);
         }

         is.close();
         cr.close();
     }

     /** This test that writes larger than the size of the buffer output
      * will correctly increment the file pointer.
      */
     public void testLargeWrites() throws IOException {
         IndexOutput os = dir.createOutput("testBufferStart.txt");

         byte[] largeBuf = new byte[2048];
         for (int i=0; i<largeBuf.length; i++) {
             largeBuf[i] = (byte) (Math.random() * 256);
         }

         long currentPos = os.getFilePointer();
         os.writeBytes(largeBuf, largeBuf.length);

         try {
             assertEquals(currentPos + largeBuf.length, os.getFilePointer());
         } finally {
             os.close();
         }

     }

    public void testAddExternalFile() throws IOException {
        createSequenceFile(dir, "d1", (byte) 0, 15);

        Directory newDir = newDirectory();
        CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
        csw.addFile("d1", dir);
        csw.close();

        CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
        IndexInput expected = dir.openInput("d1");
        IndexInput actual = csr.openInput("d1");
        assertSameStreams("d1", expected, actual);
        assertSameSeekBehavior("d1", expected, actual);
        expected.close();
        actual.close();
        csr.close();

        newDir.close();
    }

   // Make sure we don't somehow use more than 1 descriptor
   // when reading a CFS with many subs:
   public void testManySubFiles() throws IOException {

     final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
     final int FILE_COUNT = 10000;

     for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
       IndexOutput out = d.createOutput("file." + fileIdx);
       out.writeByte((byte) fileIdx);
       out.close();
     }

     final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
     for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
       cfw.addFile("file." + fileIdx);
     }
     cfw.close();

     final IndexInput[] ins = new IndexInput[FILE_COUNT];
     final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
     for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
       ins[fileIdx] = cfr.openInput("file." + fileIdx);
     }

     for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
       assertEquals((byte) fileIdx, ins[fileIdx].readByte());
     }

     for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
       ins[fileIdx].close();
     }
     cfr.close();
     d.close();
   }
 }
	package org.apache.lucene.index;

	/**
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	import java.io.IOException;
	import java.io.File;

	import org.apache.lucene.util.LuceneTestCase;
	import org.apache.lucene.store.IndexOutput;
	import org.apache.lucene.store.Directory;
	import org.apache.lucene.store.IndexInput;
	import org.apache.lucene.store.MockDirectoryWrapper;
	import org.apache.lucene.store.MockDirectoryWrapper.Failure;
	import org.apache.lucene.store.SimpleFSDirectory;
	import org.apache.lucene.store._TestHelper;
	import org.apache.lucene.util._TestUtil;


	public class TestCompoundFile extends LuceneTestCase
	{
	private Directory dir;

	@Override
	public void setUp() throws Exception {
	super.setUp();
	File file = _TestUtil.getTempDir("testIndex");
	// use a simple FSDir here, to be sure to have SimpleFSInputs
	dir = new SimpleFSDirectory(file,null);
	}

	@Override
	public void tearDown() throws Exception {
	dir.close();
	super.tearDown();
	}

	/** Creates a file of the specified size with random data. */
	private void createRandomFile(Directory dir, String name, int size)
	throws IOException
	{
	IndexOutput os = dir.createOutput(name);
	for (int i=0; i<size; i++) {
	byte b = (byte) (Math.random() * 256);
	os.writeByte(b);
	}
	os.close();
	}

	/** Creates a file of the specified size with sequential data. The first
	* byte is written as the start byte provided. All subsequent bytes are
	* computed as start + offset where offset is the number of the byte.
	*/
	private void createSequenceFile(Directory dir,
	String name,
	byte start,
	int size)
	throws IOException
	{
	IndexOutput os = dir.createOutput(name);
	for (int i=0; i < size; i++) {
	os.writeByte(start);
	start ++;
	}
	os.close();
	}


	private void assertSameStreams(String msg,
	IndexInput expected,
	IndexInput test)
	throws IOException
	{
	assertNotNull(msg + " null expected", expected);
	assertNotNull(msg + " null test", test);
	assertEquals(msg + " length", expected.length(), test.length());
	assertEquals(msg + " position", expected.getFilePointer(),
	test.getFilePointer());

	byte expectedBuffer[] = new byte[512];
	byte testBuffer[] = new byte[expectedBuffer.length];

	long remainder = expected.length() - expected.getFilePointer();
	while(remainder > 0) {
	int readLen = (int) Math.min(remainder, expectedBuffer.length);
	expected.readBytes(expectedBuffer, 0, readLen);
	test.readBytes(testBuffer, 0, readLen);
	assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
	testBuffer, 0, readLen);
	remainder -= readLen;
	}
	}


	private void assertSameStreams(String msg,
	IndexInput expected,
	IndexInput actual,
	long seekTo)
	throws IOException
	{
	if(seekTo >= 0 && seekTo < expected.length())
	{
	expected.seek(seekTo);
	actual.seek(seekTo);
	assertSameStreams(msg + ", seek(mid)", expected, actual);
	}
	}



	private void assertSameSeekBehavior(String msg,
	IndexInput expected,
	IndexInput actual)
	throws IOException
	{
	// seek to 0
	long point = 0;
	assertSameStreams(msg + ", seek(0)", expected, actual, point);

	// seek to middle
	point = expected.length() / 2l;
	assertSameStreams(msg + ", seek(mid)", expected, actual, point);

	// seek to end - 2
	point = expected.length() - 2;
	assertSameStreams(msg + ", seek(end-2)", expected, actual, point);

	// seek to end - 1
	point = expected.length() - 1;
	assertSameStreams(msg + ", seek(end-1)", expected, actual, point);

	// seek to the end
	point = expected.length();
	assertSameStreams(msg + ", seek(end)", expected, actual, point);

	// seek past end
	point = expected.length() + 1;
	assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
	}


	private void assertEqualArrays(String msg,
	byte[] expected,
	byte[] test,
	int start,
	int len)
	{
	assertNotNull(msg + " null expected", expected);
	assertNotNull(msg + " null test", test);

	for (int i=start; i<len; i++) {
	assertEquals(msg + " " + i, expected[i], test[i]);
	}
	}


	// ===========================================================
	// Tests of the basic CompoundFile functionality
	// ===========================================================


	/** This test creates compound file based on a single file.
	* Files of different sizes are tested: 0, 1, 10, 100 bytes.
	*/
	public void testSingleFile() throws IOException {
	int data[] = new int[] { 0, 1, 10, 100 };
	for (int i=0; i<data.length; i++) {
	String name = "t" + data[i];
	createSequenceFile(dir, name, (byte) 0, data[i]);
	CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
	csw.addFile(name);
	csw.close();

	CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
	IndexInput expected = dir.openInput(name);
	IndexInput actual = csr.openInput(name);
	assertSameStreams(name, expected, actual);
	assertSameSeekBehavior(name, expected, actual);
	expected.close();
	actual.close();
	csr.close();
	}
	}


	/** This test creates compound file based on two files.
	*
	*/
	public void testTwoFiles() throws IOException {
	createSequenceFile(dir, "d1", (byte) 0, 15);
	createSequenceFile(dir, "d2", (byte) 0, 114);

	CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
	csw.addFile("d1");
	csw.addFile("d2");
	csw.close();

	CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
	IndexInput expected = dir.openInput("d1");
	IndexInput actual = csr.openInput("d1");
	assertSameStreams("d1", expected, actual);
	assertSameSeekBehavior("d1", expected, actual);
	expected.close();
	actual.close();

	expected = dir.openInput("d2");
	actual = csr.openInput("d2");
	assertSameStreams("d2", expected, actual);
	assertSameSeekBehavior("d2", expected, actual);
	expected.close();
	actual.close();
	csr.close();
	}

	/** This test creates a compound file based on a large number of files of
	* various length. The file content is generated randomly. The sizes range
	* from 0 to 1Mb. Some of the sizes are selected to test the buffering
	* logic in the file reading code. For this the chunk variable is set to
	* the length of the buffer used internally by the compound file logic.
	*/
	public void testRandomFiles() throws IOException {
	// Setup the test segment
	String segment = "test";
	int chunk = 1024; // internal buffer size used by the stream
	createRandomFile(dir, segment + ".zero", 0);
	createRandomFile(dir, segment + ".one", 1);
	createRandomFile(dir, segment + ".ten", 10);
	createRandomFile(dir, segment + ".hundred", 100);
	createRandomFile(dir, segment + ".big1", chunk);
	createRandomFile(dir, segment + ".big2", chunk - 1);
	createRandomFile(dir, segment + ".big3", chunk + 1);
	createRandomFile(dir, segment + ".big4", 3 * chunk);
	createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
	createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
	createRandomFile(dir, segment + ".big7", 1000 * chunk);

	// Setup extraneous files
	createRandomFile(dir, "onetwothree", 100);
	createRandomFile(dir, segment + ".notIn", 50);
	createRandomFile(dir, segment + ".notIn2", 51);

	// Now test
	CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
	final String data[] = new String[] {
	".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
	".big4", ".big5", ".big6", ".big7"
	};
	for (int i=0; i<data.length; i++) {
	csw.addFile(segment + data[i]);
	}
	csw.close();

	CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
	for (int i=0; i<data.length; i++) {
	IndexInput check = dir.openInput(segment + data[i]);
	IndexInput test = csr.openInput(segment + data[i]);
	assertSameStreams(data[i], check, test);
	assertSameSeekBehavior(data[i], check, test);
	test.close();
	check.close();
	}
	csr.close();
	}


	/** Setup a larger compound file with a number of components, each of
	* which is a sequential file (so that we can easily tell that we are
	* reading in the right byte). The methods sets up 20 files - f0 to f19,
	* the size of each file is 1000 bytes.
	*/
	private void setUp_2() throws IOException {
	CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
	for (int i=0; i<20; i++) {
	createSequenceFile(dir, "f" + i, (byte) 0, 2000);
	cw.addFile("f" + i);
	}
	cw.close();
	}


	public void testReadAfterClose() throws IOException {
	demo_FSIndexInputBug(dir, "test");
	}

	private void demo_FSIndexInputBug(Directory fsdir, String file)
	throws IOException
	{
	// Setup the test file - we need more than 1024 bytes
	IndexOutput os = fsdir.createOutput(file);
	for(int i=0; i<2000; i++) {
	os.writeByte((byte) i);
	}
	os.close();

	IndexInput in = fsdir.openInput(file);

	// This read primes the buffer in IndexInput
	in.readByte();

	// Close the file
	in.close();

	// ERROR: this call should fail, but succeeds because the buffer
	// is still filled
	in.readByte();

	// ERROR: this call should fail, but succeeds for some reason as well
	in.seek(1099);

	try {
	// OK: this call correctly fails. We are now past the 1024 internal
	// buffer, so an actual IO is attempted, which fails
	in.readByte();
	fail("expected readByte() to throw exception");
	} catch (IOException e) {
	// expected exception
	}
	}


	static boolean isCSIndexInput(IndexInput is) {
	return is instanceof CompoundFileReader.CSIndexInput;
	}

	static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
	if (isCSIndexInput(is)) {
	CompoundFileReader.CSIndexInput cis =
	(CompoundFileReader.CSIndexInput) is;

	return _TestHelper.isSimpleFSIndexInputOpen(cis.base);
	} else {
	return false;
	}
	}


	public void testClonedStreamsClosing() throws IOException {
	setUp_2();
	CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

	// basic clone
	IndexInput expected = dir.openInput("f11");

	// this test only works for FSIndexInput
	assertTrue(_TestHelper.isSimpleFSIndexInput(expected));
	assertTrue(_TestHelper.isSimpleFSIndexInputOpen(expected));

	IndexInput one = cr.openInput("f11");
	assertTrue(isCSIndexInputOpen(one));

	IndexInput two = (IndexInput) one.clone();
	assertTrue(isCSIndexInputOpen(two));

	assertSameStreams("basic clone one", expected, one);
	expected.seek(0);
	assertSameStreams("basic clone two", expected, two);

	// Now close the first stream
	one.close();
	assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));

	// The following should really fail since we couldn't expect to
	// access a file once close has been called on it (regardless of
	// buffering and/or clone magic)
	expected.seek(0);
	two.seek(0);
	assertSameStreams("basic clone two/2", expected, two);


	// Now close the compound reader
	cr.close();
	assertFalse("Now closed one", isCSIndexInputOpen(one));
	assertFalse("Now closed two", isCSIndexInputOpen(two));

	// The following may also fail since the compound stream is closed
	expected.seek(0);
	two.seek(0);
	//assertSameStreams("basic clone two/3", expected, two);


	// Now close the second clone
	two.close();
	expected.seek(0);
	two.seek(0);
	//assertSameStreams("basic clone two/4", expected, two);

	expected.close();
	}


	/** This test opens two files from a compound stream and verifies that
	* their file positions are independent of each other.
	*/
	public void testRandomAccess() throws IOException {
	setUp_2();
	CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

	// Open two files
	IndexInput e1 = dir.openInput("f11");
	IndexInput e2 = dir.openInput("f3");

	IndexInput a1 = cr.openInput("f11");
	IndexInput a2 = dir.openInput("f3");

	// Seek the first pair
	e1.seek(100);
	a1.seek(100);
	assertEquals(100, e1.getFilePointer());
	assertEquals(100, a1.getFilePointer());
	byte be1 = e1.readByte();
	byte ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now seek the second pair
	e2.seek(1027);
	a2.seek(1027);
	assertEquals(1027, e2.getFilePointer());
	assertEquals(1027, a2.getFilePointer());
	byte be2 = e2.readByte();
	byte ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Now make sure the first one didn't move
	assertEquals(101, e1.getFilePointer());
	assertEquals(101, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now more the first one again, past the buffer length
	e1.seek(1910);
	a1.seek(1910);
	assertEquals(1910, e1.getFilePointer());
	assertEquals(1910, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now make sure the second set didn't move
	assertEquals(1028, e2.getFilePointer());
	assertEquals(1028, a2.getFilePointer());
	be2 = e2.readByte();
	ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Move the second set back, again cross the buffer size
	e2.seek(17);
	a2.seek(17);
	assertEquals(17, e2.getFilePointer());
	assertEquals(17, a2.getFilePointer());
	be2 = e2.readByte();
	ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Finally, make sure the first set didn't move
	// Now make sure the first one didn't move
	assertEquals(1911, e1.getFilePointer());
	assertEquals(1911, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	e1.close();
	e2.close();
	a1.close();
	a2.close();
	cr.close();
	}

	/** This test opens two files from a compound stream and verifies that
	* their file positions are independent of each other.
	*/
	public void testRandomAccessClones() throws IOException {
	setUp_2();
	CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

	// Open two files
	IndexInput e1 = cr.openInput("f11");
	IndexInput e2 = cr.openInput("f3");

	IndexInput a1 = (IndexInput) e1.clone();
	IndexInput a2 = (IndexInput) e2.clone();

	// Seek the first pair
	e1.seek(100);
	a1.seek(100);
	assertEquals(100, e1.getFilePointer());
	assertEquals(100, a1.getFilePointer());
	byte be1 = e1.readByte();
	byte ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now seek the second pair
	e2.seek(1027);
	a2.seek(1027);
	assertEquals(1027, e2.getFilePointer());
	assertEquals(1027, a2.getFilePointer());
	byte be2 = e2.readByte();
	byte ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Now make sure the first one didn't move
	assertEquals(101, e1.getFilePointer());
	assertEquals(101, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now more the first one again, past the buffer length
	e1.seek(1910);
	a1.seek(1910);
	assertEquals(1910, e1.getFilePointer());
	assertEquals(1910, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	// Now make sure the second set didn't move
	assertEquals(1028, e2.getFilePointer());
	assertEquals(1028, a2.getFilePointer());
	be2 = e2.readByte();
	ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Move the second set back, again cross the buffer size
	e2.seek(17);
	a2.seek(17);
	assertEquals(17, e2.getFilePointer());
	assertEquals(17, a2.getFilePointer());
	be2 = e2.readByte();
	ba2 = a2.readByte();
	assertEquals(be2, ba2);

	// Finally, make sure the first set didn't move
	// Now make sure the first one didn't move
	assertEquals(1911, e1.getFilePointer());
	assertEquals(1911, a1.getFilePointer());
	be1 = e1.readByte();
	ba1 = a1.readByte();
	assertEquals(be1, ba1);

	e1.close();
	e2.close();
	a1.close();
	a2.close();
	cr.close();
	}


	public void testFileNotFound() throws IOException {
	setUp_2();
	CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");

	// Open two files
	try {
	cr.openInput("bogus");
	fail("File not found");

	} catch (IOException e) {
	/* success */
	//System.out.println("SUCCESS: File Not Found: " + e);
	}

	cr.close();
	}


	public void testReadPastEOF() throws IOException {
	setUp_2();
	CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
	IndexInput is = cr.openInput("f2");
	is.seek(is.length() - 10);
	byte b[] = new byte[100];
	is.readBytes(b, 0, 10);

	try {
	is.readByte();
	fail("Single byte read past end of file");
	} catch (IOException e) {
	/* success */
	//System.out.println("SUCCESS: single byte read past end of file: " + e);
	}

	is.seek(is.length() - 10);
	try {
	is.readBytes(b, 0, 50);
	fail("Block read past end of file");
	} catch (IOException e) {
	/* success */
	//System.out.println("SUCCESS: block read past end of file: " + e);
	}

	is.close();
	cr.close();
	}

	/** This test that writes larger than the size of the buffer output
	* will correctly increment the file pointer.
	*/
	public void testLargeWrites() throws IOException {
	IndexOutput os = dir.createOutput("testBufferStart.txt");

	byte[] largeBuf = new byte[2048];
	for (int i=0; i<largeBuf.length; i++) {
	largeBuf[i] = (byte) (Math.random() * 256);
	}

	long currentPos = os.getFilePointer();
	os.writeBytes(largeBuf, largeBuf.length);

	try {
	assertEquals(currentPos + largeBuf.length, os.getFilePointer());
	} finally {
	os.close();
	}

	}

	public void testAddExternalFile() throws IOException {
	createSequenceFile(dir, "d1", (byte) 0, 15);

	Directory newDir = newDirectory();
	CompoundFileWriter csw = new CompoundFileWriter(newDir, "d.csf");
	csw.addFile("d1", dir);
	csw.close();

	CompoundFileReader csr = new CompoundFileReader(newDir, "d.csf");
	IndexInput expected = dir.openInput("d1");
	IndexInput actual = csr.openInput("d1");
	assertSameStreams("d1", expected, actual);
	assertSameSeekBehavior("d1", expected, actual);
	expected.close();
	actual.close();
	csr.close();

	newDir.close();
	}

	// Make sure we don't somehow use more than 1 descriptor
	// when reading a CFS with many subs:
	public void testManySubFiles() throws IOException {

	final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
	final int FILE_COUNT = 10000;

	for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
	IndexOutput out = d.createOutput("file." + fileIdx);
	out.writeByte((byte) fileIdx);
	out.close();
	}

	final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
	for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
	cfw.addFile("file." + fileIdx);
	}
	cfw.close();

	final IndexInput[] ins = new IndexInput[FILE_COUNT];
	final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
	for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
	ins[fileIdx] = cfr.openInput("file." + fileIdx);
	}

	for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
	assertEquals((byte) fileIdx, ins[fileIdx].readByte());
	}

	for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
	ins[fileIdx].close();
	}
	cfr.close();
	d.close();
	}
	}