Google Git
Sign in
apache / lucene-jira-archive / refs/heads/gh-pages / . / docs / attachments / LUCENE-2735 / LUCENE-2735_alt.patch
blob: 9146b467faa66506e5c370dd948c0e9fcfab6cd0 [file] [log] [blame]
Index: lucene/src/test/org/apache/lucene/index/codecs/gvint/TestGVIntIndexInputOutput.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/codecs/gvint/TestGVIntIndexInputOutput.java (revision 0)
+++ lucene/src/test/org/apache/lucene/index/codecs/gvint/TestGVIntIndexInputOutput.java (revision 0)
@@ -0,0 +1,62 @@
+package org.apache.lucene.index.codecs.gvint;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Simple testcase for {@link GVintIndexInput} & {@link GVintIndexOutput}
+ */
+public class TestGVIntIndexInputOutput extends LuceneTestCase {
+ private GVintFactory f = new GVintFactory();
+
+ public void testSimpleIntBlocks() throws Exception {
+ Directory dir = newDirectory();
+ IntIndexOutput out = f.createOutput(dir, "test");
+ for(int i=0;i<11777;i++) {
+ out.write(i);
+ }
+ out.close();
+
+ IntIndexInput in = f.openInput(dir, "test");
+ IntIndexInput.Reader r = in.reader();
+
+ for(int i=0;i<11777;i++) {
+ assertEquals(i, r.next());
+ }
+ in.close();
+
+ dir.close();
+ }
+
+ public void testEmptySimpleIntBlocks() throws Exception {
+ Directory dir = newDirectory();
+ IntIndexOutput out = f.createOutput(dir, "test");
+ // write no ints
+ out.close();
+
+ IntIndexInput in = f.openInput(dir, "test");
+ in.reader();
+ // read no ints
+ in.close();
+ dir.close();
+ }
+}
Property changes on: lucene/src/test/org/apache/lucene/index/codecs/gvint/TestGVIntIndexInputOutput.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Index: lucene/src/test/org/apache/lucene/index/codecs/gvint/GVintMicroBenchmark.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/codecs/gvint/GVintMicroBenchmark.java (revision 0)
+++ lucene/src/test/org/apache/lucene/index/codecs/gvint/GVintMicroBenchmark.java (revision 0)
@@ -0,0 +1,319 @@
+package org.apache.lucene.index.codecs.gvint;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.NIOFSDirectory;
+
+/**
+ */
+public class GVintMicroBenchmark {
+ private static final int[] SHIFT = { 6, 4, 2, 0 };
+
+ public static void encode(final IndexOutput out, final int... buffer) throws IOException {
+ assert buffer != null && buffer.length == 4;
+ int lengths = 0;
+ byte[] byteBuffer = new byte[17];
+ int bufferPos = 1;
+ for (int i = 0; i < 4; i++) {
+ int current = buffer[i];
+ if (((current >>> 24) & 0xFF) != 0) {
+ byteBuffer[bufferPos++] = (byte) (0xFF & current);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >>> 8);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >>> 16);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >> 24);
+ lengths |= 0xFF & (3 << SHIFT[i]);
+ } else if (((current >>> 16) & 0xFF) != 0) {
+ byteBuffer[bufferPos++] = (byte) (0xFF & current);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >>> 8);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >>> 16);
+ lengths |= 0xFF & (2 << SHIFT[i]);
+ } else if (((current >>> 8) & 0xFF) != 0) {
+ byteBuffer[bufferPos++] = (byte) (0xFF & current);
+ byteBuffer[bufferPos++] = (byte) (0xFF & current >>> 8);
+ lengths |= 0xFF & (1 << SHIFT[i]);
+ } else {
+ byteBuffer[bufferPos++] = (byte) (0xFF & current);
+ lengths |= 0xFF & (0 << SHIFT[i]);
+ }
+
+ }
+ byteBuffer[0] = (byte) (0xFF & lengths);
+ out.writeBytes(byteBuffer, bufferPos);
+ }
+
+ public static void decode(final IndexInput in, final int[] buffer)
+ throws IOException {
+ assert buffer != null && buffer.length == 4;
+ final int[] lengthTable = LOOKUP[0xFF & in.readByte()];
+ buffer[0] = readGroupInt(lengthTable[0], in);
+ buffer[1] = readGroupInt(lengthTable[1], in);
+ buffer[2] = readGroupInt(lengthTable[2], in);
+ buffer[3] = readGroupInt(lengthTable[3], in);
+ }
+
+ private static int readGroupInt(final int len, final IndexInput input) throws IOException {
+ switch (len) {
+ case 4:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8)
+ | ((input.readByte() & 0xFF) << 16)
+ | ((input.readByte() & 0xFF) << 24);
+ case 3:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8)
+ | ((input.readByte() & 0xFF) << 16);
+ case 2:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8);
+ }
+ return (input.readByte() & 0xFF);
+ }
+
+ // public static void decode1(final IndexInput in, final int[] buffer)
+ // throws IOException {
+ // assert buffer != null && buffer.length == 4;
+ // final int[] lengthTable = LOOKUP[0xFF & in.readByte()];
+ // in.readBytes(readBuffer, 0, lengthTable[4]);
+ // buffer[0] = readGroupInt(lengthTable[0], 0, readBuffer);
+ // buffer[1] = readGroupInt(lengthTable[1], lengthTable[0], readBuffer);
+ // buffer[2] = readGroupInt(lengthTable[2], lengthTable[0] + lengthTable[1],
+ // readBuffer);
+ // buffer[3] = readGroupInt(lengthTable[3], lengthTable[4] - lengthTable[3],
+ // readBuffer);
+ // }
+ // private static byte[] readBuffer = new byte[16];
+ // private static int readGroupInt(int len, int offset, byte[] readbuffer)
+ // throws IOException {
+ // switch (len) {
+ // case 4:
+ // return ((readbuffer[offset++] & 0xFF))
+ // | ((readbuffer[offset++] & 0xFF) << 8)
+ // | ((readbuffer[offset++] & 0xFF) << 16)
+ // | ((readbuffer[offset++] & 0xFF) << 24);
+ // case 3:
+ // return ((readbuffer[offset++] & 0xFF))
+ // | ((readbuffer[offset++] & 0xFF) << 8)
+ // | ((readbuffer[offset++] & 0xFF) << 16);
+ // case 2:
+ // return ((readbuffer[offset++] & 0xFF))
+ // | ((readbuffer[offset++] & 0xFF) << 8);
+ // default:
+ // return (readbuffer[offset++] & 0xFF);
+ // }
+ //
+ // }
+
+ private final static int[][] LOOKUP = { { 1, 1, 1, 1, 4 }, { 1, 1, 1, 2, 5 },
+ { 1, 1, 1, 3, 6 }, { 1, 1, 1, 4, 7 }, { 1, 1, 2, 1, 5 },
+ { 1, 1, 2, 2, 6 }, { 1, 1, 2, 3, 7 }, { 1, 1, 2, 4, 8 },
+ { 1, 1, 3, 1, 6 }, { 1, 1, 3, 2, 7 }, { 1, 1, 3, 3, 8 },
+ { 1, 1, 3, 4, 9 }, { 1, 1, 4, 1, 7 }, { 1, 1, 4, 2, 8 },
+ { 1, 1, 4, 3, 9 }, { 1, 1, 4, 4, 10 }, { 1, 2, 1, 1, 5 },
+ { 1, 2, 1, 2, 6 }, { 1, 2, 1, 3, 7 }, { 1, 2, 1, 4, 8 },
+ { 1, 2, 2, 1, 6 }, { 1, 2, 2, 2, 7 }, { 1, 2, 2, 3, 8 },
+ { 1, 2, 2, 4, 9 }, { 1, 2, 3, 1, 7 }, { 1, 2, 3, 2, 8 },
+ { 1, 2, 3, 3, 9 }, { 1, 2, 3, 4, 10 }, { 1, 2, 4, 1, 8 },
+ { 1, 2, 4, 2, 9 }, { 1, 2, 4, 3, 10 }, { 1, 2, 4, 4, 11 },
+ { 1, 3, 1, 1, 6 }, { 1, 3, 1, 2, 7 }, { 1, 3, 1, 3, 8 },
+ { 1, 3, 1, 4, 9 }, { 1, 3, 2, 1, 7 }, { 1, 3, 2, 2, 8 },
+ { 1, 3, 2, 3, 9 }, { 1, 3, 2, 4, 10 }, { 1, 3, 3, 1, 8 },
+ { 1, 3, 3, 2, 9 }, { 1, 3, 3, 3, 10 }, { 1, 3, 3, 4, 11 },
+ { 1, 3, 4, 1, 9 }, { 1, 3, 4, 2, 10 }, { 1, 3, 4, 3, 11 },
+ { 1, 3, 4, 4, 12 }, { 1, 4, 1, 1, 7 }, { 1, 4, 1, 2, 8 },
+ { 1, 4, 1, 3, 9 }, { 1, 4, 1, 4, 10 }, { 1, 4, 2, 1, 8 },
+ { 1, 4, 2, 2, 9 }, { 1, 4, 2, 3, 10 }, { 1, 4, 2, 4, 11 },
+ { 1, 4, 3, 1, 9 }, { 1, 4, 3, 2, 10 }, { 1, 4, 3, 3, 11 },
+ { 1, 4, 3, 4, 12 }, { 1, 4, 4, 1, 10 }, { 1, 4, 4, 2, 11 },
+ { 1, 4, 4, 3, 12 }, { 1, 4, 4, 4, 13 }, { 2, 1, 1, 1, 5 },
+ { 2, 1, 1, 2, 6 }, { 2, 1, 1, 3, 7 }, { 2, 1, 1, 4, 8 },
+ { 2, 1, 2, 1, 6 }, { 2, 1, 2, 2, 7 }, { 2, 1, 2, 3, 8 },
+ { 2, 1, 2, 4, 9 }, { 2, 1, 3, 1, 7 }, { 2, 1, 3, 2, 8 },
+ { 2, 1, 3, 3, 9 }, { 2, 1, 3, 4, 10 }, { 2, 1, 4, 1, 8 },
+ { 2, 1, 4, 2, 9 }, { 2, 1, 4, 3, 10 }, { 2, 1, 4, 4, 11 },
+ { 2, 2, 1, 1, 6 }, { 2, 2, 1, 2, 7 }, { 2, 2, 1, 3, 8 },
+ { 2, 2, 1, 4, 9 }, { 2, 2, 2, 1, 7 }, { 2, 2, 2, 2, 8 },
+ { 2, 2, 2, 3, 9 }, { 2, 2, 2, 4, 10 }, { 2, 2, 3, 1, 8 },
+ { 2, 2, 3, 2, 9 }, { 2, 2, 3, 3, 10 }, { 2, 2, 3, 4, 11 },
+ { 2, 2, 4, 1, 9 }, { 2, 2, 4, 2, 10 }, { 2, 2, 4, 3, 11 },
+ { 2, 2, 4, 4, 12 }, { 2, 3, 1, 1, 7 }, { 2, 3, 1, 2, 8 },
+ { 2, 3, 1, 3, 9 }, { 2, 3, 1, 4, 10 }, { 2, 3, 2, 1, 8 },
+ { 2, 3, 2, 2, 9 }, { 2, 3, 2, 3, 10 }, { 2, 3, 2, 4, 11 },
+ { 2, 3, 3, 1, 9 }, { 2, 3, 3, 2, 10 }, { 2, 3, 3, 3, 11 },
+ { 2, 3, 3, 4, 12 }, { 2, 3, 4, 1, 10 }, { 2, 3, 4, 2, 11 },
+ { 2, 3, 4, 3, 12 }, { 2, 3, 4, 4, 13 }, { 2, 4, 1, 1, 8 },
+ { 2, 4, 1, 2, 9 }, { 2, 4, 1, 3, 10 }, { 2, 4, 1, 4, 11 },
+ { 2, 4, 2, 1, 9 }, { 2, 4, 2, 2, 10 }, { 2, 4, 2, 3, 11 },
+ { 2, 4, 2, 4, 12 }, { 2, 4, 3, 1, 10 }, { 2, 4, 3, 2, 11 },
+ { 2, 4, 3, 3, 12 }, { 2, 4, 3, 4, 13 }, { 2, 4, 4, 1, 11 },
+ { 2, 4, 4, 2, 12 }, { 2, 4, 4, 3, 13 }, { 2, 4, 4, 4, 14 },
+ { 3, 1, 1, 1, 6 }, { 3, 1, 1, 2, 7 }, { 3, 1, 1, 3, 8 },
+ { 3, 1, 1, 4, 9 }, { 3, 1, 2, 1, 7 }, { 3, 1, 2, 2, 8 },
+ { 3, 1, 2, 3, 9 }, { 3, 1, 2, 4, 10 }, { 3, 1, 3, 1, 8 },
+ { 3, 1, 3, 2, 9 }, { 3, 1, 3, 3, 10 }, { 3, 1, 3, 4, 11 },
+ { 3, 1, 4, 1, 9 }, { 3, 1, 4, 2, 10 }, { 3, 1, 4, 3, 11 },
+ { 3, 1, 4, 4, 12 }, { 3, 2, 1, 1, 7 }, { 3, 2, 1, 2, 8 },
+ { 3, 2, 1, 3, 9 }, { 3, 2, 1, 4, 10 }, { 3, 2, 2, 1, 8 },
+ { 3, 2, 2, 2, 9 }, { 3, 2, 2, 3, 10 }, { 3, 2, 2, 4, 11 },
+ { 3, 2, 3, 1, 9 }, { 3, 2, 3, 2, 10 }, { 3, 2, 3, 3, 11 },
+ { 3, 2, 3, 4, 12 }, { 3, 2, 4, 1, 10 }, { 3, 2, 4, 2, 11 },
+ { 3, 2, 4, 3, 12 }, { 3, 2, 4, 4, 13 }, { 3, 3, 1, 1, 8 },
+ { 3, 3, 1, 2, 9 }, { 3, 3, 1, 3, 10 }, { 3, 3, 1, 4, 11 },
+ { 3, 3, 2, 1, 9 }, { 3, 3, 2, 2, 10 }, { 3, 3, 2, 3, 11 },
+ { 3, 3, 2, 4, 12 }, { 3, 3, 3, 1, 10 }, { 3, 3, 3, 2, 11 },
+ { 3, 3, 3, 3, 12 }, { 3, 3, 3, 4, 13 }, { 3, 3, 4, 1, 11 },
+ { 3, 3, 4, 2, 12 }, { 3, 3, 4, 3, 13 }, { 3, 3, 4, 4, 14 },
+ { 3, 4, 1, 1, 9 }, { 3, 4, 1, 2, 10 }, { 3, 4, 1, 3, 11 },
+ { 3, 4, 1, 4, 12 }, { 3, 4, 2, 1, 10 }, { 3, 4, 2, 2, 11 },
+ { 3, 4, 2, 3, 12 }, { 3, 4, 2, 4, 13 }, { 3, 4, 3, 1, 11 },
+ { 3, 4, 3, 2, 12 }, { 3, 4, 3, 3, 13 }, { 3, 4, 3, 4, 14 },
+ { 3, 4, 4, 1, 12 }, { 3, 4, 4, 2, 13 }, { 3, 4, 4, 3, 14 },
+ { 3, 4, 4, 4, 15 }, { 4, 1, 1, 1, 7 }, { 4, 1, 1, 2, 8 },
+ { 4, 1, 1, 3, 9 }, { 4, 1, 1, 4, 10 }, { 4, 1, 2, 1, 8 },
+ { 4, 1, 2, 2, 9 }, { 4, 1, 2, 3, 10 }, { 4, 1, 2, 4, 11 },
+ { 4, 1, 3, 1, 9 }, { 4, 1, 3, 2, 10 }, { 4, 1, 3, 3, 11 },
+ { 4, 1, 3, 4, 12 }, { 4, 1, 4, 1, 10 }, { 4, 1, 4, 2, 11 },
+ { 4, 1, 4, 3, 12 }, { 4, 1, 4, 4, 13 }, { 4, 2, 1, 1, 8 },
+ { 4, 2, 1, 2, 9 }, { 4, 2, 1, 3, 10 }, { 4, 2, 1, 4, 11 },
+ { 4, 2, 2, 1, 9 }, { 4, 2, 2, 2, 10 }, { 4, 2, 2, 3, 11 },
+ { 4, 2, 2, 4, 12 }, { 4, 2, 3, 1, 10 }, { 4, 2, 3, 2, 11 },
+ { 4, 2, 3, 3, 12 }, { 4, 2, 3, 4, 13 }, { 4, 2, 4, 1, 11 },
+ { 4, 2, 4, 2, 12 }, { 4, 2, 4, 3, 13 }, { 4, 2, 4, 4, 14 },
+ { 4, 3, 1, 1, 9 }, { 4, 3, 1, 2, 10 }, { 4, 3, 1, 3, 11 },
+ { 4, 3, 1, 4, 12 }, { 4, 3, 2, 1, 10 }, { 4, 3, 2, 2, 11 },
+ { 4, 3, 2, 3, 12 }, { 4, 3, 2, 4, 13 }, { 4, 3, 3, 1, 11 },
+ { 4, 3, 3, 2, 12 }, { 4, 3, 3, 3, 13 }, { 4, 3, 3, 4, 14 },
+ { 4, 3, 4, 1, 12 }, { 4, 3, 4, 2, 13 }, { 4, 3, 4, 3, 14 },
+ { 4, 3, 4, 4, 15 }, { 4, 4, 1, 1, 10 }, { 4, 4, 1, 2, 11 },
+ { 4, 4, 1, 3, 12 }, { 4, 4, 1, 4, 13 }, { 4, 4, 2, 1, 11 },
+ { 4, 4, 2, 2, 12 }, { 4, 4, 2, 3, 13 }, { 4, 4, 2, 4, 14 },
+ { 4, 4, 3, 1, 12 }, { 4, 4, 3, 2, 13 }, { 4, 4, 3, 3, 14 },
+ { 4, 4, 3, 4, 15 }, { 4, 4, 4, 1, 13 }, { 4, 4, 4, 2, 14 },
+ { 4, 4, 4, 3, 15 }, { 4, 4, 4, 4, 16 }, };
+
+ // run with -server -Xmx512M -Xms512M
+ public static void main(String[] args) throws IOException {
+ Random r = new Random();
+ int[][] intsToEncode = new int[1000000][];
+
+ for (int maxShift = 4; maxShift <= 30; maxShift += 1) {
+ int max = 1 << maxShift; // 127;
+ for (int i = 0; i < intsToEncode.length; i++) {
+
+ if (intsToEncode[i] == null) {
+ intsToEncode[i] = new int[4];
+
+ }
+ intsToEncode[i][0] = r.nextInt(max);
+ intsToEncode[i][1] = r.nextInt(max);
+ intsToEncode[i][2] = r.nextInt(max);
+ intsToEncode[i][3] = r.nextInt(max);
+ }
+ long gvi = 0;
+ long vints = 0;
+ // warmup
+ for (int i = 0; i < 3; i++) {
+ groupintsTest(intsToEncode);
+ vintsTest(intsToEncode);
+ }
+ for (int i = 0; i < 10; i++) {
+ if (i % 2 == 0) {
+ gvi += groupintsTest(intsToEncode);
+ vints += vintsTest(intsToEncode);
+ } else {
+ vints += vintsTest(intsToEncode);
+ gvi += groupintsTest(intsToEncode);
+ }
+ }
+ System.out.println("Running 4 Million random ints with max value: "
+ + (max));
+ System.out.println("GroupVarInt time per value: " + (gvi / 10)
+ / (intsToEncode.length * 4) + "ns - time to decode 4M ints: "
+ + TimeUnit.MILLISECONDS.convert(gvi / 10, TimeUnit.NANOSECONDS)
+ + "ms");
+
+ System.out.println("Vint time per value: " + (vints / 10)
+ / (intsToEncode.length * 4) + "ns - time to decode 4M ints: "
+ + TimeUnit.MILLISECONDS.convert(vints / 10, TimeUnit.NANOSECONDS)
+ + "ms");
+
+ System.out.println();
+ }
+
+ }
+
+ private static long vintsTest(int[][] intsToEncode) throws IOException {
+ File f = new File("/tmp/vint");
+ if (!f.exists())
+ f.mkdir();
+ final Directory dir = new NIOFSDirectory(f);
+ final IndexOutput out = dir.createOutput("vint.dat");
+ for (int[] is : intsToEncode) {
+ for (int j = 0; j < is.length; j++) {
+ out.writeVInt(is[j]);
+ }
+ }
+ out.flush();
+ out.close();
+ final IndexInput input = dir.openInput("vint.dat");
+ long t = System.nanoTime();
+ for (int[] is : intsToEncode) {
+ int[] buffer = new int[4];
+ for (int j = 0; j < buffer.length; j++) {
+ buffer[j] = input.readVInt();
+ }
+ assert Arrays.equals(buffer, is);
+ }
+ input.close();
+ dir.deleteFile("vint.dat");
+ dir.close();
+ return System.nanoTime() - t;
+ }
+
+ private static long groupintsTest(int[][] intsToEncode) throws IOException {
+ File f = new File("/tmp/gvi");
+ if (!f.exists())
+ f.mkdir();
+ final Directory dir = new NIOFSDirectory(f);
+ final IndexOutput out = dir.createOutput("groupvint.dat");
+ for (int[] is : intsToEncode) {
+ encode(out, is);
+ }
+ out.flush();
+ out.close();
+
+ final IndexInput input = dir.openInput("groupvint.dat");
+ int[] decode = new int[4];
+ long t = System.nanoTime();
+ for (int i = 0; i < intsToEncode.length; i++) {
+ decode(input, decode);
+ assert Arrays.equals(decode, intsToEncode[i]) : Arrays.toString(decode)
+ + " " + Arrays.toString(intsToEncode[i]) + " int num: " + i;
+ }
+ input.close();
+ dir.deleteFile("groupvint.dat");
+ dir.close();
+ return System.nanoTime() - t;
+ }
+
+}
Property changes on: lucene/src/test/org/apache/lucene/index/codecs/gvint/GVintMicroBenchmark.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Index: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintFactory.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintFactory.java (revision 0)
+++ lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintFactory.java (revision 0)
@@ -0,0 +1,42 @@
+package org.apache.lucene.index.codecs.gvint;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.lucene.index.codecs.sep.IntIndexInput;
+import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.store.Directory;
+
+/**
+ * {@link IntStreamFactory} for Group VarInt
+ */
+public final class GVintFactory extends IntStreamFactory {
+
+ @Override
+ public IntIndexInput openInput(Directory dir, String fileName,
+ int readBufferSize) throws IOException {
+ return new GVintIndexInput(dir, fileName, 1024);
+ }
+
+ @Override
+ public IntIndexOutput createOutput(Directory dir, String fileName)
+ throws IOException {
+ return new GVintIndexOutput(dir, fileName);
+ }
+
+}
Property changes on: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintFactory.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Index: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexInput.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexInput.java (revision 0)
+++ lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexInput.java (revision 0)
@@ -0,0 +1,79 @@
+package org.apache.lucene.index.codecs.gvint;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.CodecUtil;
+
+/**
+ * Reads IndexInputs written with {@link GVintIndexOutput}.
+ *
+ * @lucene.experimental
+ */
+public final class GVintIndexInput extends FixedIntBlockIndexInput {
+
+ public GVintIndexInput(Directory dir, String fileName, int readBufferSize)
+ throws IOException {
+ super(createInput(dir, fileName, readBufferSize));
+
+ }
+
+ private static final IndexInput createInput(Directory dir, String fileName,
+ int readBufferSize) throws IOException {
+ IndexInput input = dir.openInput(fileName, readBufferSize);
+ CodecUtil.checkHeader(input, GVintIndexOutput.CODEC,
+ GVintIndexOutput.VERSION_START, GVintIndexOutput.VERSION_START);
+ return input;
+ }
+
+ @Override
+ protected BlockReader getBlockReader(final IndexInput in, final int[] buffer)
+ throws IOException {
+ return new BlockReader() {
+ public void seek(long pos) {
+ }
+ public void readBlock() throws IOException {
+ final int lengths = 0xFF & in.readByte();
+ buffer[0] = readGroupInt((lengths & 0x03), in);
+ buffer[1] = readGroupInt(((lengths>>2) & 0x03), in);
+ buffer[2] = readGroupInt(((lengths>>4) & 0x03), in);
+ buffer[3] = readGroupInt(((lengths>>6)), in);
+ }
+ };
+ }
+
+ private static int readGroupInt(int len, IndexInput input) throws IOException {
+ switch ((byte)len) {
+ case 0:
+ return (input.readByte() & 0xFF);
+ case 1:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8);
+ case 2:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8)
+ | ((input.readByte() & 0xFF) << 16);
+ default:
+ return ((input.readByte() & 0xFF)) | ((input.readByte() & 0xFF) << 8)
+ | ((input.readByte() & 0xFF) << 16)
+ | ((input.readByte()) << 24);
+ }
+ }
+}
Property changes on: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexInput.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Index: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexOutput.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexOutput.java (revision 0)
+++ lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexOutput.java (revision 0)
@@ -0,0 +1,79 @@
+package org.apache.lucene.index.codecs.gvint;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.CodecUtil;
+
+/**
+ * Writes ints as Group VarInt
+ *
+ * @lucene.experimental
+ */
+public final class GVintIndexOutput extends FixedIntBlockIndexOutput {
+ final static String CODEC = "GVInt";
+ final static int VERSION_START = 0;
+ final static int VERSION_CURRENT = VERSION_START;
+
+ public GVintIndexOutput(Directory dir, String fileName) throws IOException {
+ super(createOutput(dir, fileName), 4); // use block size of 4 for now
+ }
+
+ private static IndexOutput createOutput(Directory dir, String fileName)
+ throws IOException {
+ IndexOutput output = dir.createOutput(fileName);
+ CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT);
+ return output;
+ }
+
+ @Override
+ protected void flushBlock() throws IOException {
+ assert buffer != null && buffer.length == 4;
+ int lengths = 0;
+ byte[] byteBuffer = new byte[17]; // TODO: get rid of this allocation if possible
+ int bufferPos = 1;
+ for (int i = 0; i < 4; i++) {
+ final int current = buffer[i];
+ byte b;
+ if (current <= 0xff) {
+ byteBuffer[bufferPos++] = (byte) (current);
+ } else if ((b=(byte)(current >>> 8)) != 0) {
+ byteBuffer[bufferPos++] = (byte) (current);
+ byteBuffer[bufferPos++] = b;
+ lengths |= (1 << (i<<1));
+ } else if ((b=(byte)(current >>> 16)) != 0) {
+ byteBuffer[bufferPos++] = (byte) (current);
+ byteBuffer[bufferPos++] = (byte) (current >>> 8);
+ byteBuffer[bufferPos++] = b;
+ lengths |= (2 << (i<<1));
+ } else if ((b=(byte)(current >>> 24)) != 0) {
+ byteBuffer[bufferPos++] = (byte) (current);
+ byteBuffer[bufferPos++] = (byte) (current >>> 8);
+ byteBuffer[bufferPos++] = (byte) (current >>> 16);
+ byteBuffer[bufferPos++] = b;
+ lengths |= (3 << (i<<1));
+ }
+ }
+ byteBuffer[0] = (byte)lengths;
+ out.writeBytes(byteBuffer, bufferPos);
+ }
+}
Property changes on: lucene/src/java/org/apache/lucene/index/codecs/gvint/GVintIndexOutput.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native