lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.util.fst;


import java.util.Arrays;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;

public class TestBytesStore extends LuceneTestCase {

  public void testRandom() throws Exception {

    final int iters = atLeast(10);
    final int maxBytes = TEST_NIGHTLY ? 200000 : 20000;
    for(int iter=0;iter<iters;iter++) {
      final int numBytes = TestUtil.nextInt(random(), 1, maxBytes);
      final byte[] expected = new byte[numBytes];
      final int blockBits = TestUtil.nextInt(random(), 8, 15);
      final BytesStore bytes = new BytesStore(blockBits);
      if (VERBOSE) {
        System.out.println("TEST: iter=" + iter + " numBytes=" + numBytes + " blockBits=" + blockBits);
      }

      int pos = 0;
      while(pos < numBytes) {
        int op = random().nextInt(8);
        if (VERBOSE) {
          System.out.println("  cycle pos=" + pos);
        }
        switch(op) {

        case 0:
          {
            // write random byte
            byte b = (byte) random().nextInt(256);
            if (VERBOSE) {
              System.out.println("    writeByte b=" + b);
            }

            expected[pos++] = b;
            bytes.writeByte(b);
          }
          break;

        case 1:
          {
            // write random byte[]
            int len = random().nextInt(Math.min(numBytes - pos, 100));
            byte[] temp = new byte[len];
            random().nextBytes(temp);
            if (VERBOSE) {
              System.out.println("    writeBytes len=" + len + " bytes=" + Arrays.toString(temp));
            }
            System.arraycopy(temp, 0, expected, pos, temp.length);
            bytes.writeBytes(temp, 0, temp.length);
            pos += len;
          }
          break;

        case 2:
          {
            // write int @ absolute pos
            if (pos > 4) {
              int x = random().nextInt();
              int randomPos = random().nextInt(pos-4);
              if (VERBOSE) {
                System.out.println("    abs writeInt pos=" + randomPos + " x=" + x);
              }
              bytes.writeInt(randomPos, x);
              expected[randomPos++] = (byte) (x >> 24);
              expected[randomPos++] = (byte) (x >> 16);
              expected[randomPos++] = (byte) (x >> 8);
              expected[randomPos++] = (byte) x;
            }
          }
          break;

        case 3:
          {
            // reverse bytes
            if (pos > 1) {
              int len = TestUtil.nextInt(random(), 2, Math.min(100, pos));
              int start;
              if (len == pos) {
                start = 0;
              } else {
                start = random().nextInt(pos - len);
              }
              int end = start + len - 1;
              if (VERBOSE) {
                System.out.println("    reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos);
              }
              bytes.reverse(start, end);

              while(start <= end) {
                byte b = expected[end];
                expected[end] = expected[start];
                expected[start] = b;
                start++;
                end--;
              }
            }
          }
          break;

        case 4:
          {
            // abs write random byte[]
            if (pos > 2) {
              int randomPos = random().nextInt(pos-1);
              int len = TestUtil.nextInt(random(), 1, Math.min(pos - randomPos - 1, 100));
              byte[] temp = new byte[len];
              random().nextBytes(temp);
              if (VERBOSE) {
                System.out.println("    abs writeBytes pos=" + randomPos + " len=" + len + " bytes=" + Arrays.toString(temp));
              }
              System.arraycopy(temp, 0, expected, randomPos, temp.length);
              bytes.writeBytes(randomPos, temp, 0, temp.length);
            }
          }
          break;

        case 5:
          {
            // copyBytes
            if (pos > 1) {
              int src = random().nextInt(pos-1);
              int dest = TestUtil.nextInt(random(), src + 1, pos - 1);
              int len = TestUtil.nextInt(random(), 1, Math.min(300, pos - dest));
              if (VERBOSE) {
                System.out.println("    copyBytes src=" + src + " dest=" + dest + " len=" + len);
              }
              System.arraycopy(expected, src, expected, dest, len);
              bytes.copyBytes(src, dest, len);
            }
          }
          break;

        case 6:
          {
            // skip
            int len = random().nextInt(Math.min(100, numBytes - pos));

            if (VERBOSE) {
              System.out.println("    skip len=" + len);
            }

            pos += len;
            bytes.skipBytes(len);

            // NOTE: must fill in zeros in case truncate was
            // used, else we get false fails:
            if (len > 0) {
              byte[] zeros = new byte[len];
              bytes.writeBytes(pos-len, zeros, 0, len);
            }
          }
          break;

        case 7:
          {
            // absWriteByte
            if (pos > 0) {
              int dest = random().nextInt(pos);
              byte b = (byte) random().nextInt(256);
              expected[dest] = b;
              bytes.writeByte(dest, b);
            }
            break;
          }
        }

        assertEquals(pos, bytes.getPosition());

        if (pos > 0 && random().nextInt(50) == 17) {
          // truncate
          int len = TestUtil.nextInt(random(), 1, Math.min(pos, 100));
          bytes.truncate(pos - len);
          pos -= len;
          Arrays.fill(expected, pos, pos+len, (byte) 0);
          if (VERBOSE) {
            System.out.println("    truncate len=" + len + " newPos=" + pos);
          }
        }

        if ((pos > 0 && random().nextInt(200) == 17)) {
          verify(bytes, expected, pos);
        }
      }

      BytesStore bytesToVerify;

      if (random().nextBoolean()) {
        if (VERBOSE) {
          System.out.println("TEST: save/load final bytes");
        }
        Directory dir = newDirectory();
        IndexOutput out = dir.createOutput("bytes", IOContext.DEFAULT);
        bytes.writeTo(out);
        out.close();
        IndexInput in = dir.openInput("bytes", IOContext.DEFAULT);
        bytesToVerify = new BytesStore(in, numBytes, TestUtil.nextInt(random(), 256, Integer.MAX_VALUE));
        in.close();
        dir.close();
      } else {
        bytesToVerify = bytes;
      }

      verify(bytesToVerify, expected, numBytes);
    }
  }

  private void verify(BytesStore bytes, byte[] expected, int totalLength) throws Exception {
    assertEquals(totalLength, bytes.getPosition());
    if (totalLength == 0) {
      return;
    }
    if (VERBOSE) {
      System.out.println("  verify...");
    }
    
    // First verify whole thing in one blast:
    byte[] actual = new byte[totalLength];
    if (random().nextBoolean()) {
      if (VERBOSE) {
        System.out.println("    bulk: reversed");
      }
      // reversed
      FST.BytesReader r = bytes.getReverseReader();
      assertTrue(r.reversed());
      r.setPosition(totalLength-1);
      r.readBytes(actual, 0, actual.length);
      int start = 0;
      int end = totalLength - 1;
      while(start < end) {
        byte b = actual[start];
        actual[start] = actual[end];
        actual[end] = b;
        start++;
        end--;
      }
    } else {
      // forward
      if (VERBOSE) {
        System.out.println("    bulk: forward");
      }
      FST.BytesReader r = bytes.getForwardReader();
      assertFalse(r.reversed());
      r.readBytes(actual, 0, actual.length);
    }

    for(int i=0;i<totalLength;i++) {
      assertEquals("byte @ index=" + i, expected[i], actual[i]);
    }

    FST.BytesReader r;

    // Then verify ops:
    boolean reversed = random().nextBoolean();
    if (reversed) {
      if (VERBOSE) {
        System.out.println("    ops: reversed");
      }
      r = bytes.getReverseReader();
    } else {
      if (VERBOSE) {
        System.out.println("    ops: forward");
      }
      r = bytes.getForwardReader();
    }

    if (totalLength > 1) {
      int numOps = TestUtil.nextInt(random(), 100, 200);
      for(int op=0;op<numOps;op++) {

        int numBytes = random().nextInt(Math.min(1000, totalLength-1));
        int pos;
        if (reversed) {
          pos = TestUtil.nextInt(random(), numBytes, totalLength - 1);
        } else {
          pos = random().nextInt(totalLength-numBytes);
        }
        if (VERBOSE) {
          System.out.println("    op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos);
        }
        byte[] temp = new byte[numBytes];
        r.setPosition(pos);
        assertEquals(pos, r.getPosition());
        r.readBytes(temp, 0, temp.length);
        for(int i=0;i<numBytes;i++) {
          byte expectedByte;
          if (reversed) {
            expectedByte = expected[pos - i];
          } else {
            expectedByte = expected[pos + i];
          }
          assertEquals("byte @ index=" + i, expectedByte, temp[i]);
        }

        int left;
        int expectedPos;

        if (reversed) {
          expectedPos = pos-numBytes;
          left = (int) r.getPosition();
        } else {
          expectedPos = pos+numBytes;
          left = (int) (totalLength - r.getPosition());
        }
        assertEquals(expectedPos, r.getPosition());

        if (left > 4) {
          int skipBytes = random().nextInt(left-4);

          int expectedInt = 0;
          if (reversed) {
            expectedPos -= skipBytes;
            expectedInt |= (expected[expectedPos--]&0xFF)<<24;
            expectedInt |= (expected[expectedPos--]&0xFF)<<16;
            expectedInt |= (expected[expectedPos--]&0xFF)<<8;
            expectedInt |= (expected[expectedPos--]&0xFF);
          } else {
            expectedPos += skipBytes;
            expectedInt |= (expected[expectedPos++]&0xFF)<<24;
            expectedInt |= (expected[expectedPos++]&0xFF)<<16;
            expectedInt |= (expected[expectedPos++]&0xFF)<<8;
            expectedInt |= (expected[expectedPos++]&0xFF);
          }

          if (VERBOSE) {
            System.out.println("    skip numBytes=" + skipBytes);
            System.out.println("    readInt");
          }

          r.skipBytes(skipBytes);
          assertEquals(expectedInt, r.readInt());
        }
      }
    }
  }
}